From 04b311c69dc0f0b8d1aa7f7ef247153f27bdeee9 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 9 Aug 2023 22:57:49 -0500 Subject: [PATCH 01/16] Add port config to deployment spec. --- app/deployment_create.py | 95 +++++++++++++++++++++++++++------------- 1 file changed, 64 insertions(+), 31 deletions(-) diff --git a/app/deployment_create.py b/app/deployment_create.py index af5eefb2..452dd2c3 100644 --- a/app/deployment_create.py +++ b/app/deployment_create.py @@ -27,10 +27,27 @@ from app.deploy_types import DeploymentContext, DeployCommandContext def _make_default_deployment_dir(): return "deployment-001" +def _get_ports(stack): + ports = {} + parsed_stack = get_parsed_stack_config(stack) + pods = parsed_stack["pods"] + yaml = get_yaml() + for pod in pods: + pod_file_path = os.path.join(get_compose_file_dir(), f"docker-compose-{pod}.yml") + parsed_pod_file = yaml.load(open(pod_file_path, "r")) + if "services" in parsed_pod_file: + for svc_name, svc in parsed_pod_file["services"].items(): + if "ports" in svc: + normalized = [ str(x) for x in svc["ports"] ] + if pod in ports: + ports[pod][svc_name] = normalized + else: + ports[pod] = { svc_name: normalized } + return ports def _get_named_volumes(stack): # Parse the compose files looking for named volumes - named_volumes = [] + named_volumes = {} parsed_stack = get_parsed_stack_config(stack) pods = parsed_stack["pods"] yaml = get_yaml() @@ -39,10 +56,9 @@ def _get_named_volumes(stack): parsed_pod_file = yaml.load(open(pod_file_path, "r")) if "volumes" in parsed_pod_file: volumes = parsed_pod_file["volumes"] - for volume in volumes.keys(): - # Volume definition looks like: - # 'laconicd-data': None - named_volumes.append(volume) + # Volume definition looks like: + # 'laconicd-data': None + named_volumes[pod] = list(volumes.keys()) return named_volumes @@ -61,25 +77,30 @@ def _create_bind_dir_if_relative(volume, path_string, compose_dir): # See: https://stackoverflow.com/questions/45699189/editing-docker-compose-yml-with-pyyaml -def _fixup_pod_file(pod, spec, compose_dir): - # Fix up volumes - if "volumes" in spec: - spec_volumes = spec["volumes"] - if "volumes" in pod: - pod_volumes = pod["volumes"] - for volume in pod_volumes.keys(): - if volume in spec_volumes: - volume_spec = spec_volumes[volume] - volume_spec_fixedup = volume_spec if Path(volume_spec).is_absolute() else f".{volume_spec}" - _create_bind_dir_if_relative(volume, volume_spec, compose_dir) - new_volume_spec = {"driver": "local", - "driver_opts": { - "type": "none", - "device": volume_spec_fixedup, - "o": "bind" - } - } - pod["volumes"][volume] = new_volume_spec +def _fixup_pod_file(pod_name, pod, spec, compose_dir): + if pod_name in spec["pods"]: + # Fix up volumes + if "volumes" in spec["pods"][pod_name]: + spec_volumes = spec["pods"][pod_name]["volumes"] + if "volumes" in pod: + pod_volumes = pod["volumes"] + for volume in pod_volumes.keys(): + if volume in spec_volumes: + volume_spec = spec_volumes[volume] + volume_spec_fixedup = volume_spec if Path(volume_spec).is_absolute() else f".{volume_spec}" + _create_bind_dir_if_relative(volume, volume_spec, compose_dir) + new_volume_spec = {"driver": "local", + "driver_opts": { + "type": "none", + "device": volume_spec_fixedup, + "o": "bind" + } + } + pod["volumes"][volume] = new_volume_spec + # Fix up ports + if "ports" in spec["pods"][pod_name]: + for container_name, container_ports in spec["pods"][pod_name]["ports"].items(): + pod["services"][container_name]["ports"] = container_ports def call_stack_deploy_init(deploy_command_context): @@ -148,12 +169,24 @@ def init(ctx, output): spec_file_content.update(default_spec_file_content) if verbose: print(f"Creating spec file for stack: {stack}") - named_volumes = _get_named_volumes(stack) - if named_volumes: - volume_descriptors = {} - for named_volume in named_volumes: - volume_descriptors[named_volume] = f"./data/{named_volume}" - spec_file_content["volumes"] = volume_descriptors + + parsed_stack = get_parsed_stack_config(stack) + pods = dict([ (p, {}) for p in parsed_stack["pods"]]) + + named_volumes_by_pod = _get_named_volumes(stack) + if named_volumes_by_pod: + for pod_name in named_volumes_by_pod: + volume_descriptors = {} + for named_volume in named_volumes_by_pod[pod_name]: + volume_descriptors[named_volume] = f"./data/{named_volume}" + pods[pod_name]["volumes"] = volume_descriptors + + ports_by_pod = _get_ports(stack) + if ports_by_pod: + for pod_name in ports_by_pod: + pods[pod_name]["ports"] = ports_by_pod[pod_name] + + spec_file_content["pods"] = pods with open(output, "w") as output_file: yaml.dump(spec_file_content, output_file) @@ -191,7 +224,7 @@ def create(ctx, spec_file, deployment_dir): extra_config_dirs = _find_extra_config_dirs(parsed_pod_file, pod) if global_options(ctx).debug: print(f"extra config dirs: {extra_config_dirs}") - _fixup_pod_file(parsed_pod_file, parsed_spec, destination_compose_dir) + _fixup_pod_file(pod, parsed_pod_file, parsed_spec, destination_compose_dir) with open(os.path.join(destination_compose_dir, os.path.basename(pod_file_path)), "w") as output_file: yaml.dump(parsed_pod_file, output_file) # Copy the config files for the pod, if any -- 2.45.2 From 671a1444eb3fc729e805e56316e9350f89c2e0c2 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Thu, 10 Aug 2023 23:12:35 -0500 Subject: [PATCH 02/16] Update mainnet stack. --- .../docker-compose-mainnet-eth-keycloak.yml | 35 + .../docker-compose-mainnet-eth-metrics.yml | 23 + .../compose/docker-compose-mainnet-eth.yml | 19 +- .../import/cerc-realm.json | 2087 +++++++++++++++++ .../config/mainnet-eth-keycloak/keycloak.env | 17 + .../nginx/keycloak_proxy.conf | 72 + .../grafana/etc/dashboards/eth_dashboard.json | 1409 +++++++++++ .../provisioning/dashboards/dashboards.yml | 9 + .../provisioning/datasources/prometheus.yml | 19 + .../prometheus/etc/prometheus.yml | 19 + app/data/config/mainnet-eth/geth.env | 57 + app/data/config/mainnet-eth/lighthouse.env | 30 + .../config/mainnet-eth/scripts/run-geth.sh | 55 +- .../mainnet-eth/scripts/run-lighthouse.sh | 32 +- app/data/stacks/mainnet-eth/stack.yml | 3 + 15 files changed, 3841 insertions(+), 45 deletions(-) create mode 100644 app/data/compose/docker-compose-mainnet-eth-keycloak.yml create mode 100644 app/data/compose/docker-compose-mainnet-eth-metrics.yml create mode 100644 app/data/config/mainnet-eth-keycloak/import/cerc-realm.json create mode 100644 app/data/config/mainnet-eth-keycloak/keycloak.env create mode 100644 app/data/config/mainnet-eth-keycloak/nginx/keycloak_proxy.conf create mode 100644 app/data/config/mainnet-eth-metrics/grafana/etc/dashboards/eth_dashboard.json create mode 100644 app/data/config/mainnet-eth-metrics/grafana/etc/provisioning/dashboards/dashboards.yml create mode 100644 app/data/config/mainnet-eth-metrics/grafana/etc/provisioning/datasources/prometheus.yml create mode 100644 app/data/config/mainnet-eth-metrics/prometheus/etc/prometheus.yml create mode 100644 app/data/config/mainnet-eth/geth.env create mode 100644 app/data/config/mainnet-eth/lighthouse.env diff --git a/app/data/compose/docker-compose-mainnet-eth-keycloak.yml b/app/data/compose/docker-compose-mainnet-eth-keycloak.yml new file mode 100644 index 00000000..c4ebdea8 --- /dev/null +++ b/app/data/compose/docker-compose-mainnet-eth-keycloak.yml @@ -0,0 +1,35 @@ +version: '3.8' + +services: + keycloak-db: + image: postgres:14-alpine + env_file: + - ../config/mainnet-eth-keycloak/keycloak.env + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "5432"] + interval: 30s + timeout: 10s + retries: 10 + start_period: 3s + volumes: + - mainnet_eth_keycloak_db:/var/lib/postgresql/data + ports: + - 5432 + + keycloak: + image: cerc/keycloak:local + env_file: + - ../config/mainnet-eth-keycloak/keycloak.env + environment: + JAVA_OPTS_APPEND: "-Dkeycloak.migration.action=import -Dkeycloak.migration.provider=dir -Dkeycloak.migration.dir=/import -Dkeycloak.migration.strategy=IGNORE_EXISTING" + volumes: + - ../config/mainnet-eth-keycloak/import:/import + ports: + - 8080 + command: ["start"] + depends_on: + keycloak-db: + condition: service_healthy + +volumes: + mainnet_eth_keycloak_db: \ No newline at end of file diff --git a/app/data/compose/docker-compose-mainnet-eth-metrics.yml b/app/data/compose/docker-compose-mainnet-eth-metrics.yml new file mode 100644 index 00000000..c3978018 --- /dev/null +++ b/app/data/compose/docker-compose-mainnet-eth-metrics.yml @@ -0,0 +1,23 @@ +version: "3.2" +services: + prometheus: + restart: always + image: prom/prometheus + depends_on: + mainnet-eth-geth-1: + condition: service_healthy + volumes: + - ../config/mainnet-eth-metrics/prometheus/etc:/etc/prometheus + ports: + - "9090" + grafana: + restart: always + image: grafana/grafana + environment: + - GF_SECURITY_ADMIN_PASSWORD=changeme6325 + volumes: + - ../config/mainnet-eth-metrics/grafana/etc/provisioning/dashboards:/etc/grafana/provisioning/dashboards + - ../config/mainnet-eth-metrics/grafana/etc/provisioning/datasources:/etc/grafana/provisioning/datasources + - ../config/mainnet-eth-metrics/grafana/etc/dashboards:/etc/grafana/dashboards + ports: + - "3000" diff --git a/app/data/compose/docker-compose-mainnet-eth.yml b/app/data/compose/docker-compose-mainnet-eth.yml index 7b44a88e..3a76b593 100644 --- a/app/data/compose/docker-compose-mainnet-eth.yml +++ b/app/data/compose/docker-compose-mainnet-eth.yml @@ -6,16 +6,13 @@ services: hostname: mainnet-eth-geth-1 cap_add: - SYS_PTRACE - environment: - CERC_REMOTE_DEBUG: "true" - CERC_RUN_STATEDIFF: ${CERC_RUN_STATEDIFF:-detect} - CERC_STATEDIFF_DB_NODE_ID: 1 - CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} image: cerc/go-ethereum:local entrypoint: /bin/sh command: -c "/opt/run-geth.sh" + env_file: + - ../config/mainnet-eth/geth.env volumes: - - mainnet_eth_geth_1_data:/root/ethdata + - mainnet_eth_geth_1_data:/data - mainnet_eth_config_data:/etc/mainnet-eth - ../config/mainnet-eth/scripts/run-geth.sh:/opt/run-geth.sh healthcheck: @@ -26,6 +23,7 @@ services: start_period: 3s ports: - "8545" + - "8551" - "40000" - "6060" @@ -39,16 +37,19 @@ services: retries: 10 start_period: 30s environment: - EXECUTION_ENDPOINT: "http://mainnet-eth-geth-1:8551" + LIGHTHOUSE_EXECUTION_ENDPOINT: "http://mainnet-eth-geth-1:8551" + env_file: + - ../config/mainnet-eth/lighthouse.env image: cerc/lighthouse:local entrypoint: /bin/sh command: -c "/opt/run-lighthouse.sh" volumes: - - mainnet_eth_lighthouse_1_data:/var/lighthouse-data-dir + - mainnet_eth_lighthouse_1_data:/data - mainnet_eth_config_data:/etc/mainnet-eth - ../config/mainnet-eth/scripts/run-lighthouse.sh:/opt/run-lighthouse.sh ports: - - "8001" + - "5052" + - "9000" volumes: mainnet_eth_config_data: diff --git a/app/data/config/mainnet-eth-keycloak/import/cerc-realm.json b/app/data/config/mainnet-eth-keycloak/import/cerc-realm.json new file mode 100644 index 00000000..e1e9dc97 --- /dev/null +++ b/app/data/config/mainnet-eth-keycloak/import/cerc-realm.json @@ -0,0 +1,2087 @@ +{ + "id": "cerc", + "realm": "cerc", + "notBefore": 0, + "defaultSignatureAlgorithm": "RS256", + "revokeRefreshToken": false, + "refreshTokenMaxReuse": 0, + "accessTokenLifespan": 300, + "accessTokenLifespanForImplicitFlow": 900, + "ssoSessionIdleTimeout": 1800, + "ssoSessionMaxLifespan": 36000, + "ssoSessionIdleTimeoutRememberMe": 0, + "ssoSessionMaxLifespanRememberMe": 0, + "offlineSessionIdleTimeout": 2592000, + "offlineSessionMaxLifespanEnabled": false, + "offlineSessionMaxLifespan": 5184000, + "clientSessionIdleTimeout": 0, + "clientSessionMaxLifespan": 0, + "clientOfflineSessionIdleTimeout": 0, + "clientOfflineSessionMaxLifespan": 0, + "accessCodeLifespan": 60, + "accessCodeLifespanUserAction": 300, + "accessCodeLifespanLogin": 1800, + "actionTokenGeneratedByAdminLifespan": 43200, + "actionTokenGeneratedByUserLifespan": 300, + "oauth2DeviceCodeLifespan": 600, + "oauth2DevicePollingInterval": 5, + "enabled": true, + "sslRequired": "external", + "registrationAllowed": false, + "registrationEmailAsUsername": false, + "rememberMe": false, + "verifyEmail": false, + "loginWithEmailAllowed": true, + "duplicateEmailsAllowed": false, + "resetPasswordAllowed": false, + "editUsernameAllowed": false, + "bruteForceProtected": false, + "permanentLockout": false, + "maxFailureWaitSeconds": 900, + "minimumQuickLoginWaitSeconds": 60, + "waitIncrementSeconds": 60, + "quickLoginCheckMilliSeconds": 1000, + "maxDeltaTimeSeconds": 43200, + "failureFactor": 30, + "defaultRole": { + "id": "211646ea-04a3-467e-9f25-f7539a405d03", + "name": "default-roles-cerc", + "description": "${role_default-roles}", + "composite": true, + "clientRole": false, + "containerId": "cerc" + }, + "requiredCredentials": [ + "password" + ], + "otpPolicyType": "totp", + "otpPolicyAlgorithm": "HmacSHA1", + "otpPolicyInitialCounter": 0, + "otpPolicyDigits": 6, + "otpPolicyLookAheadWindow": 1, + "otpPolicyPeriod": 30, + "otpPolicyCodeReusable": false, + "otpSupportedApplications": [ + "totpAppGoogleName", + "totpAppFreeOTPName" + ], + "webAuthnPolicyRpEntityName": "keycloak", + "webAuthnPolicySignatureAlgorithms": [ + "ES256" + ], + "webAuthnPolicyRpId": "", + "webAuthnPolicyAttestationConveyancePreference": "not specified", + "webAuthnPolicyAuthenticatorAttachment": "not specified", + "webAuthnPolicyRequireResidentKey": "not specified", + "webAuthnPolicyUserVerificationRequirement": "not specified", + "webAuthnPolicyCreateTimeout": 0, + "webAuthnPolicyAvoidSameAuthenticatorRegister": false, + "webAuthnPolicyAcceptableAaguids": [], + "webAuthnPolicyPasswordlessRpEntityName": "keycloak", + "webAuthnPolicyPasswordlessSignatureAlgorithms": [ + "ES256" + ], + "webAuthnPolicyPasswordlessRpId": "", + "webAuthnPolicyPasswordlessAttestationConveyancePreference": "not specified", + "webAuthnPolicyPasswordlessAuthenticatorAttachment": "not specified", + "webAuthnPolicyPasswordlessRequireResidentKey": "not specified", + "webAuthnPolicyPasswordlessUserVerificationRequirement": "not specified", + "webAuthnPolicyPasswordlessCreateTimeout": 0, + "webAuthnPolicyPasswordlessAvoidSameAuthenticatorRegister": false, + "webAuthnPolicyPasswordlessAcceptableAaguids": [], + "users": [ + { + "id": "70af487b-c6d8-4f51-84d2-a23e8c9df7a3", + "createdTimestamp": 1670910521308, + "username": "service-account-dashboard-client", + "enabled": true, + "totp": false, + "emailVerified": false, + "serviceAccountClientId": "dashboard-client", + "disableableCredentialTypes": [], + "requiredActions": [], + "notBefore": 0 + } + ], + "scopeMappings": [ + { + "clientScope": "offline_access", + "roles": [ + "offline_access" + ] + } + ], + "clientScopeMappings": { + "account": [ + { + "client": "account-console", + "roles": [ + "manage-account", + "view-groups" + ] + } + ] + }, + "clients": [ + { + "id": "1ff40495-e44c-4cbc-886a-87c3ca1edc9d", + "clientId": "account", + "name": "${client_account}", + "rootUrl": "${authBaseUrl}", + "baseUrl": "/realms/cerc/account/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "secret": "**********", + "redirectUris": [ + "/realms/cerc/account/*" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "post.logout.redirect.uris": "+" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "profile", + "roles", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "75b3bc74-dd4d-4d0a-940c-f1a809c004a6", + "clientId": "account-console", + "name": "${client_account-console}", + "rootUrl": "${authBaseUrl}", + "baseUrl": "/realms/cerc/account/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "/realms/cerc/account/*" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "post.logout.redirect.uris": "+", + "pkce.code.challenge.method": "S256" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "protocolMappers": [ + { + "id": "4ec0bc59-9111-46da-a7d3-549b7aa0e398", + "name": "audience resolve", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-resolve-mapper", + "consentRequired": false, + "config": {} + } + ], + "defaultClientScopes": [ + "web-origins", + "profile", + "roles", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "0dc11c0a-b159-4e48-bdf9-31a1fccd25c6", + "clientId": "admin-cli", + "name": "${client_admin-cli}", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "secret": "**********", + "redirectUris": [], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": false, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "post.logout.redirect.uris": "+" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "profile", + "roles", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "c8a751e8-08be-427f-9191-3bdc0cc3e829", + "clientId": "broker", + "name": "${client_broker}", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "secret": "**********", + "redirectUris": [], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "post.logout.redirect.uris": "+" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "profile", + "roles", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "2d7384c7-9301-4a57-8fb5-b42aa43b8d3f", + "clientId": "dashboard-client", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "secret": "**********", + "redirectUris": [ + "http://localhost:8180/*" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": true, + "authorizationServicesEnabled": true, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "saml.assertion.signature": "false", + "saml.force.post.binding": "false", + "saml.multivalued.roles": "false", + "saml.encrypt": "false", + "post.logout.redirect.uris": "+", + "saml.server.signature": "false", + "saml.server.signature.keyinfo.ext": "false", + "exclude.session.state.from.auth.response": "false", + "saml_force_name_id_format": "false", + "saml.client.signature": "false", + "tls.client.certificate.bound.access.tokens": "false", + "saml.authnstatement": "false", + "display.on.consent.screen": "false", + "saml.onetimeuse.condition": "false" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": true, + "nodeReRegistrationTimeout": -1, + "protocolMappers": [ + { + "id": "5746e878-a248-4170-9f6e-221dad215e25", + "name": "Client ID", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": false, + "config": { + "user.session.note": "clientId", + "userinfo.token.claim": "true", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "clientId", + "jsonType.label": "String" + } + }, + { + "id": "e584082b-a232-45bd-8520-bc88908642a1", + "name": "Client IP Address", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": false, + "config": { + "user.session.note": "clientAddress", + "userinfo.token.claim": "true", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "clientAddress", + "jsonType.label": "String" + } + }, + { + "id": "bd9eaacb-6c5b-4bf1-bc0d-2457f7f7a767", + "name": "api-key", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "api-key", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "api-key", + "jsonType.label": "String" + } + }, + { + "id": "a10834b6-005a-4083-84e7-69ea2c08c0a8", + "name": "Client Host", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": false, + "config": { + "user.session.note": "clientHost", + "userinfo.token.claim": "true", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "clientHost", + "jsonType.label": "String" + } + } + ], + "defaultClientScopes": [ + "web-origins", + "profile", + "roles", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ], + "authorizationSettings": { + "allowRemoteResourceManagement": true, + "policyEnforcementMode": "ENFORCING", + "resources": [ + { + "name": "Default Resource", + "type": "urn:dashboard-client:resources:default", + "ownerManagedAccess": false, + "attributes": {}, + "_id": "fd85dada-073c-4da0-ac3c-73a823e86e70", + "uris": [ + "/*" + ] + } + ], + "policies": [], + "scopes": [], + "decisionStrategy": "UNANIMOUS" + } + }, + { + "id": "1a91181f-823b-4cbf-9d7a-f5f097a00d73", + "clientId": "realm-management", + "name": "${client_realm-management}", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "secret": "**********", + "redirectUris": [], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": true, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "post.logout.redirect.uris": "+" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "profile", + "roles", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "1c10f8e8-6553-4d39-a705-8380214a01c9", + "clientId": "security-admin-console", + "name": "${client_security-admin-console}", + "rootUrl": "${authAdminUrl}", + "baseUrl": "/admin/cerc/console/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "secret": "**********", + "redirectUris": [ + "/admin/cerc/console/*" + ], + "webOrigins": [ + "+" + ], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "post.logout.redirect.uris": "+", + "pkce.code.challenge.method": "S256" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "protocolMappers": [ + { + "id": "e65eaf73-6a5d-44da-a129-930481351e5e", + "name": "locale", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "locale", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "locale", + "jsonType.label": "String" + } + } + ], + "defaultClientScopes": [ + "web-origins", + "profile", + "roles", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + } + ], + "clientScopes": [ + { + "id": "6e3c0398-187d-4515-9fad-e09225e6484c", + "name": "email", + "description": "OpenID Connect built-in scope: email", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${emailScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "7e81f77f-8631-46a0-979a-7744ea451880", + "name": "email", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "email", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "email", + "jsonType.label": "String" + } + }, + { + "id": "b41d73c7-5ae4-4492-9f05-fe737bbd8a9b", + "name": "email verified", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "emailVerified", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "email_verified", + "jsonType.label": "boolean" + } + } + ] + }, + { + "id": "42c276ef-e93e-4e65-a963-b84a7b229449", + "name": "microprofile-jwt", + "description": "Microprofile - JWT built-in scope", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "false" + }, + "protocolMappers": [ + { + "id": "145a68c2-075a-417c-bafb-824c0bb02dd2", + "name": "groups", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "multivalued": "true", + "userinfo.token.claim": "true", + "user.attribute": "foo", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "groups", + "jsonType.label": "String" + } + }, + { + "id": "d9f7cb53-ae29-47e0-aaf8-edd40acfa5b9", + "name": "upn", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "username", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "upn", + "jsonType.label": "String" + } + } + ] + }, + { + "id": "c88a720f-8fe6-4750-81b6-b87551066905", + "name": "role_list", + "description": "SAML role list", + "protocol": "saml", + "attributes": { + "consent.screen.text": "${samlRoleListScopeConsentText}", + "display.on.consent.screen": "true" + }, + "protocolMappers": [ + { + "id": "0244f0c4-773e-40e3-a0e4-308f5b10ab78", + "name": "role list", + "protocol": "saml", + "protocolMapper": "saml-role-list-mapper", + "consentRequired": false, + "config": { + "single": "false", + "attribute.nameformat": "Basic", + "attribute.name": "Role" + } + } + ] + }, + { + "id": "ba66e4d5-12f9-4c44-921d-42d901485803", + "name": "web-origins", + "description": "OpenID Connect scope for add allowed web origins to the access token", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "false", + "consent.screen.text": "" + }, + "protocolMappers": [ + { + "id": "0f2f1ccf-7292-4e49-a079-d9166ec100bb", + "name": "allowed web origins", + "protocol": "openid-connect", + "protocolMapper": "oidc-allowed-origins-mapper", + "consentRequired": false, + "config": {} + } + ] + }, + { + "id": "e73a3670-4958-43bc-b5fa-982a895bc8d4", + "name": "phone", + "description": "OpenID Connect built-in scope: phone", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${phoneScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "bf04e15d-711a-4f66-b6f4-c35f21fcb0c8", + "name": "phone number", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "phoneNumber", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "phone_number", + "jsonType.label": "String" + } + }, + { + "id": "76493880-66bf-40d9-8f41-b14a8d400b1d", + "name": "phone number verified", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "phoneNumberVerified", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "phone_number_verified", + "jsonType.label": "boolean" + } + } + ] + }, + { + "id": "5cb4b2c4-880e-4437-b905-19a5eb471765", + "name": "profile", + "description": "OpenID Connect built-in scope: profile", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${profileScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "f7ba27e0-141e-4389-93d2-cc6c5fb1f78a", + "name": "family name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "lastName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "family_name", + "jsonType.label": "String" + } + }, + { + "id": "f3c2b39e-a11b-4640-acb3-c6ce139235e5", + "name": "zoneinfo", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "zoneinfo", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "zoneinfo", + "jsonType.label": "String" + } + }, + { + "id": "bbf1c241-15c1-4d94-812a-ad4e260f77df", + "name": "picture", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "picture", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "picture", + "jsonType.label": "String" + } + }, + { + "id": "0dc161e8-f2e8-4017-b895-c24a78d38e92", + "name": "birthdate", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "birthdate", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "birthdate", + "jsonType.label": "String" + } + }, + { + "id": "86761664-57a4-47df-a891-d0d721243327", + "name": "nickname", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "nickname", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "nickname", + "jsonType.label": "String" + } + }, + { + "id": "20f086d5-a07c-4711-88aa-3396fafb2adf", + "name": "full name", + "protocol": "openid-connect", + "protocolMapper": "oidc-full-name-mapper", + "consentRequired": false, + "config": { + "id.token.claim": "true", + "access.token.claim": "true", + "userinfo.token.claim": "true" + } + }, + { + "id": "d79a8b71-9312-4658-b14b-8f3145052116", + "name": "username", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "username", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "preferred_username", + "jsonType.label": "String" + } + }, + { + "id": "ad342e65-e36a-48cc-a90a-d48aacefab01", + "name": "profile", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "profile", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "profile", + "jsonType.label": "String" + } + }, + { + "id": "6abd60fb-39b7-4063-aaee-5ff380f0a97e", + "name": "gender", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "gender", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "gender", + "jsonType.label": "String" + } + }, + { + "id": "580133fc-8e44-4e7a-a526-dcbc7d82c911", + "name": "website", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "website", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "website", + "jsonType.label": "String" + } + }, + { + "id": "4bcde3c0-41ef-45e6-a23b-aea222640399", + "name": "locale", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "locale", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "locale", + "jsonType.label": "String" + } + }, + { + "id": "1ed7844e-9002-4c7b-be3d-61f9b3c725b9", + "name": "middle name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "middleName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "middle_name", + "jsonType.label": "String" + } + }, + { + "id": "4c9e9ec5-f40d-4b6e-9385-f86b0d228940", + "name": "updated at", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "updatedAt", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "updated_at", + "jsonType.label": "String" + } + }, + { + "id": "bf10082b-d485-4cf4-bf31-f0181884e8cf", + "name": "given name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "firstName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "given_name", + "jsonType.label": "String" + } + } + ] + }, + { + "id": "b25abfe5-1130-4d7d-98f4-227f8b0dc4f9", + "name": "roles", + "description": "OpenID Connect scope for add user roles to the access token", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "true", + "consent.screen.text": "${rolesScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "84b22a06-dced-4b2f-bbc8-f818b01c73d0", + "name": "realm roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "user.attribute": "foo", + "access.token.claim": "true", + "claim.name": "realm_access.roles", + "jsonType.label": "String", + "multivalued": "true" + } + }, + { + "id": "5c6ed3cf-0840-4191-81ea-7092569f70fe", + "name": "audience resolve", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-resolve-mapper", + "consentRequired": false, + "config": {} + }, + { + "id": "dce34b2a-e58f-41b8-86ab-794edeccae40", + "name": "client roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-client-role-mapper", + "consentRequired": false, + "config": { + "user.attribute": "foo", + "access.token.claim": "true", + "claim.name": "resource_access.${client_id}.roles", + "jsonType.label": "String", + "multivalued": "true" + } + } + ] + }, + { + "id": "00476d55-cd2f-4f60-92dd-6f3ff634799e", + "name": "acr", + "description": "OpenID Connect scope for add acr (authentication context class reference) to the token", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "false" + }, + "protocolMappers": [ + { + "id": "f0ae1247-2120-4513-b1d1-fab7cfecfbb8", + "name": "acr loa level", + "protocol": "openid-connect", + "protocolMapper": "oidc-acr-mapper", + "consentRequired": false, + "config": { + "id.token.claim": "true", + "access.token.claim": "true", + "userinfo.token.claim": "true" + } + } + ] + }, + { + "id": "3f68af4c-10e8-4351-a62d-f829b9832037", + "name": "address", + "description": "OpenID Connect built-in scope: address", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${addressScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "3037e6e9-e1d7-492c-a060-9b2c35c688cb", + "name": "address", + "protocol": "openid-connect", + "protocolMapper": "oidc-address-mapper", + "consentRequired": false, + "config": { + "user.attribute.formatted": "formatted", + "user.attribute.country": "country", + "user.attribute.postal_code": "postal_code", + "userinfo.token.claim": "true", + "user.attribute.street": "street", + "id.token.claim": "true", + "user.attribute.region": "region", + "access.token.claim": "true", + "user.attribute.locality": "locality" + } + } + ] + }, + { + "id": "f900704b-5e92-451e-b093-02286cc22774", + "name": "offline_access", + "description": "OpenID Connect built-in scope: offline_access", + "protocol": "openid-connect", + "attributes": { + "consent.screen.text": "${offlineAccessScopeConsentText}", + "display.on.consent.screen": "true" + } + } + ], + "defaultDefaultClientScopes": [ + "profile", + "email", + "roles", + "web-origins", + "role_list", + "acr" + ], + "defaultOptionalClientScopes": [ + "address", + "microprofile-jwt", + "phone", + "offline_access" + ], + "browserSecurityHeaders": { + "contentSecurityPolicyReportOnly": "", + "xContentTypeOptions": "nosniff", + "xRobotsTag": "none", + "xFrameOptions": "SAMEORIGIN", + "contentSecurityPolicy": "frame-src 'self'; frame-ancestors 'self'; object-src 'none';", + "xXSSProtection": "1; mode=block", + "strictTransportSecurity": "max-age=31536000; includeSubDomains" + }, + "smtpServer": {}, + "accountTheme": "custom", + "eventsEnabled": true, + "eventsExpiration": 604800, + "eventsListeners": [ + "api-key-registration-generation", + "metrics-listener", + "jboss-logging" + ], + "enabledEventTypes": [ + "SEND_RESET_PASSWORD", + "REMOVE_TOTP", + "REVOKE_GRANT", + "UPDATE_TOTP", + "LOGIN_ERROR", + "CLIENT_LOGIN", + "RESET_PASSWORD_ERROR", + "IMPERSONATE_ERROR", + "CODE_TO_TOKEN_ERROR", + "CUSTOM_REQUIRED_ACTION", + "RESTART_AUTHENTICATION", + "IMPERSONATE", + "UPDATE_PROFILE_ERROR", + "LOGIN", + "UPDATE_PASSWORD_ERROR", + "CLIENT_INITIATED_ACCOUNT_LINKING", + "TOKEN_EXCHANGE", + "LOGOUT", + "REGISTER", + "CLIENT_REGISTER", + "IDENTITY_PROVIDER_LINK_ACCOUNT", + "UPDATE_PASSWORD", + "CLIENT_DELETE", + "FEDERATED_IDENTITY_LINK_ERROR", + "IDENTITY_PROVIDER_FIRST_LOGIN", + "CLIENT_DELETE_ERROR", + "VERIFY_EMAIL", + "CLIENT_LOGIN_ERROR", + "RESTART_AUTHENTICATION_ERROR", + "EXECUTE_ACTIONS", + "REMOVE_FEDERATED_IDENTITY_ERROR", + "TOKEN_EXCHANGE_ERROR", + "PERMISSION_TOKEN", + "SEND_IDENTITY_PROVIDER_LINK_ERROR", + "EXECUTE_ACTION_TOKEN_ERROR", + "SEND_VERIFY_EMAIL", + "EXECUTE_ACTIONS_ERROR", + "REMOVE_FEDERATED_IDENTITY", + "IDENTITY_PROVIDER_POST_LOGIN", + "IDENTITY_PROVIDER_LINK_ACCOUNT_ERROR", + "UPDATE_EMAIL", + "REGISTER_ERROR", + "REVOKE_GRANT_ERROR", + "EXECUTE_ACTION_TOKEN", + "LOGOUT_ERROR", + "UPDATE_EMAIL_ERROR", + "CLIENT_UPDATE_ERROR", + "UPDATE_PROFILE", + "CLIENT_REGISTER_ERROR", + "FEDERATED_IDENTITY_LINK", + "SEND_IDENTITY_PROVIDER_LINK", + "SEND_VERIFY_EMAIL_ERROR", + "RESET_PASSWORD", + "CLIENT_INITIATED_ACCOUNT_LINKING_ERROR", + "REMOVE_TOTP_ERROR", + "VERIFY_EMAIL_ERROR", + "SEND_RESET_PASSWORD_ERROR", + "CLIENT_UPDATE", + "CUSTOM_REQUIRED_ACTION_ERROR", + "IDENTITY_PROVIDER_POST_LOGIN_ERROR", + "UPDATE_TOTP_ERROR", + "CODE_TO_TOKEN", + "IDENTITY_PROVIDER_FIRST_LOGIN_ERROR" + ], + "adminEventsEnabled": false, + "adminEventsDetailsEnabled": false, + "identityProviders": [], + "identityProviderMappers": [], + "components": { + "org.keycloak.services.clientregistration.policy.ClientRegistrationPolicy": [ + { + "id": "e108ed9d-422a-4c84-af0e-d7ea9ddc1890", + "name": "Trusted Hosts", + "providerId": "trusted-hosts", + "subType": "anonymous", + "subComponents": {}, + "config": { + "host-sending-registration-request-must-match": [ + "true" + ], + "client-uris-must-match": [ + "true" + ] + } + }, + { + "id": "b9a4a7f7-d684-45bd-b4bf-646be1f79364", + "name": "Full Scope Disabled", + "providerId": "scope", + "subType": "anonymous", + "subComponents": {}, + "config": {} + }, + { + "id": "8df4222e-0b62-44dc-be51-f27d828f0f66", + "name": "Allowed Protocol Mapper Types", + "providerId": "allowed-protocol-mappers", + "subType": "anonymous", + "subComponents": {}, + "config": { + "allowed-protocol-mapper-types": [ + "saml-user-property-mapper", + "oidc-sha256-pairwise-sub-mapper", + "oidc-full-name-mapper", + "saml-role-list-mapper", + "oidc-usermodel-property-mapper", + "oidc-usermodel-attribute-mapper", + "saml-user-attribute-mapper", + "oidc-address-mapper" + ] + } + }, + { + "id": "59dd3e18-4dbe-4054-b012-423e8c4da909", + "name": "Max Clients Limit", + "providerId": "max-clients", + "subType": "anonymous", + "subComponents": {}, + "config": { + "max-clients": [ + "200" + ] + } + }, + { + "id": "7ce212c8-2587-4f6c-8824-705eabb7f925", + "name": "Allowed Client Scopes", + "providerId": "allowed-client-templates", + "subType": "authenticated", + "subComponents": {}, + "config": { + "allow-default-scopes": [ + "true" + ] + } + }, + { + "id": "4cbfdd25-6c33-4bad-8d88-9a1aec6c8e25", + "name": "Consent Required", + "providerId": "consent-required", + "subType": "anonymous", + "subComponents": {}, + "config": {} + }, + { + "id": "bd008843-3c81-4750-ae85-a5e4e181b877", + "name": "Allowed Protocol Mapper Types", + "providerId": "allowed-protocol-mappers", + "subType": "authenticated", + "subComponents": {}, + "config": { + "allowed-protocol-mapper-types": [ + "oidc-usermodel-property-mapper", + "saml-role-list-mapper", + "saml-user-property-mapper", + "oidc-full-name-mapper", + "oidc-address-mapper", + "oidc-sha256-pairwise-sub-mapper", + "saml-user-attribute-mapper", + "oidc-usermodel-attribute-mapper" + ] + } + }, + { + "id": "2edf8e74-e1b6-4e6d-83a3-c1123d462d14", + "name": "Allowed Client Scopes", + "providerId": "allowed-client-templates", + "subType": "anonymous", + "subComponents": {}, + "config": { + "allow-default-scopes": [ + "true" + ] + } + } + ], + "org.keycloak.userprofile.UserProfileProvider": [ + { + "id": "bfd8d11c-d90c-4620-802d-2b5bb04ed9d3", + "providerId": "declarative-user-profile", + "subComponents": {}, + "config": {} + } + ], + "org.keycloak.keys.KeyProvider": [ + { + "id": "ca2afd56-df5d-47ab-bea4-4416c859a338", + "name": "hmac-generated", + "providerId": "hmac-generated", + "subComponents": {}, + "config": { + "priority": [ + "100" + ], + "algorithm": [ + "HS256" + ] + } + }, + { + "id": "c72d323d-5737-4bed-bbc9-41be440e99fb", + "name": "rsa-generated", + "providerId": "rsa-generated", + "subComponents": {}, + "config": { + "priority": [ + "100" + ] + } + }, + { + "id": "f80ab6e7-1b0a-4de4-acaa-3275d3f867a2", + "name": "aes-generated", + "providerId": "aes-generated", + "subComponents": {}, + "config": { + "priority": [ + "100" + ] + } + } + ] + }, + "internationalizationEnabled": false, + "supportedLocales": [], + "authenticationFlows": [ + { + "id": "43505ad9-3c8d-4f11-9f90-55bcf19e621b", + "alias": "Handle Existing Account", + "description": "Handle what to do if there is existing account with same email/username like authenticated identity provider", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "idp-confirm-link", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 20, + "autheticatorFlow": true, + "flowAlias": "Handle Existing Account - Alternatives - 0", + "userSetupAllowed": false + } + ] + }, + { + "id": "f5a8bcf1-b58f-4fd9-a0c1-4ec3933d9d64", + "alias": "Handle Existing Account - Alternatives - 0", + "description": "Subflow of Handle Existing Account with alternative executions", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "idp-email-verification", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorFlow": true, + "requirement": "ALTERNATIVE", + "priority": 20, + "autheticatorFlow": true, + "flowAlias": "Verify Existing Account by Re-authentication", + "userSetupAllowed": false + } + ] + }, + { + "id": "b3f19451-b375-4341-8c23-f9a3b531ceb0", + "alias": "Verify Existing Account by Re-authentication", + "description": "Reauthentication of existing account", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "idp-username-password-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 20, + "autheticatorFlow": true, + "flowAlias": "Verify Existing Account by Re-authentication - auth-otp-form - Conditional", + "userSetupAllowed": false + } + ] + }, + { + "id": "0db81a1c-dd36-4721-89e4-19dc7e204b56", + "alias": "Verify Existing Account by Re-authentication - auth-otp-form - Conditional", + "description": "Flow to determine if the auth-otp-form authenticator should be used or not.", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "auth-otp-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + } + ] + }, + { + "id": "e0937686-c0c4-41b2-8abd-98b5219e1953", + "alias": "browser", + "description": "browser based authentication", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "auth-cookie", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "auth-spnego", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "identity-provider-redirector", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 25, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorFlow": true, + "requirement": "ALTERNATIVE", + "priority": 30, + "autheticatorFlow": true, + "flowAlias": "forms", + "userSetupAllowed": false + } + ] + }, + { + "id": "3508fa7b-a459-44ad-b56a-af9737ed86a5", + "alias": "browser plus basic", + "description": "browser based authentication", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": false, + "authenticationExecutions": [ + { + "authenticator": "auth-cookie", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "basic-auth", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "auth-spnego", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 25, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "identity-provider-redirector", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 30, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorFlow": true, + "requirement": "ALTERNATIVE", + "priority": 31, + "autheticatorFlow": true, + "flowAlias": "browser plus basic forms", + "userSetupAllowed": false + } + ] + }, + { + "id": "79ee49ad-20f2-4967-a9bf-ddca82c1516c", + "alias": "browser plus basic forms", + "description": "Username, password, otp and other auth forms.", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": false, + "authenticationExecutions": [ + { + "authenticator": "auth-username-password-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 20, + "autheticatorFlow": true, + "flowAlias": "browser plus basic forms - auth-otp-form - Conditional", + "userSetupAllowed": false + } + ] + }, + { + "id": "802ce2dc-dd4a-45e6-837e-fecc17affe55", + "alias": "browser plus basic forms - auth-otp-form - Conditional", + "description": "Flow to determine if the auth-otp-form authenticator should be used or not.", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": false, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "auth-otp-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + } + ] + }, + { + "id": "0f4a4d19-db06-409b-baa8-a3c8a6f52a22", + "alias": "clients", + "description": "Base authentication for clients", + "providerId": "client-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "client-secret", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "client-jwt", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "client-secret-jwt", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 30, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "client-x509", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 40, + "autheticatorFlow": false, + "userSetupAllowed": false + } + ] + }, + { + "id": "b177d3f1-dad8-4b40-ac1d-04038f0e5a7d", + "alias": "direct grant", + "description": "OpenID Connect Resource Owner Grant", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "direct-grant-validate-username", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "direct-grant-validate-password", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 30, + "autheticatorFlow": true, + "flowAlias": "direct grant - direct-grant-validate-otp - Conditional", + "userSetupAllowed": false + } + ] + }, + { + "id": "788ccbc9-c3c8-468d-8d4c-d2eb04b438a5", + "alias": "direct grant - direct-grant-validate-otp - Conditional", + "description": "Flow to determine if the direct-grant-validate-otp authenticator should be used or not.", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "direct-grant-validate-otp", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + } + ] + }, + { + "id": "8edd3a8f-7d9d-4029-8fd2-21a8ead2b090", + "alias": "docker auth", + "description": "Used by Docker clients to authenticate against the IDP", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "docker-http-basic-authenticator", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + } + ] + }, + { + "id": "a67bc8ee-b99a-409f-adf5-a7d4c7f27512", + "alias": "first broker login", + "description": "Actions taken after first broker login with identity provider account, which is not yet linked to any Keycloak account", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticatorConfig": "review profile config", + "authenticator": "idp-review-profile", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 20, + "autheticatorFlow": true, + "flowAlias": "first broker login - Alternatives - 0", + "userSetupAllowed": false + } + ] + }, + { + "id": "ffe8dad9-6998-4358-ab2c-061cf7235d53", + "alias": "first broker login - Alternatives - 0", + "description": "Subflow of first broker login with alternative executions", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticatorConfig": "create unique user config", + "authenticator": "idp-create-user-if-unique", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorFlow": true, + "requirement": "ALTERNATIVE", + "priority": 20, + "autheticatorFlow": true, + "flowAlias": "Handle Existing Account", + "userSetupAllowed": false + } + ] + }, + { + "id": "26133bdd-6657-449d-a823-73519956b272", + "alias": "forms", + "description": "Username, password, otp and other auth forms.", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "auth-username-password-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 20, + "autheticatorFlow": true, + "flowAlias": "forms - auth-otp-form - Conditional", + "userSetupAllowed": false + } + ] + }, + { + "id": "57620e5a-f7cd-4e88-ac51-d78e91ff7868", + "alias": "forms - auth-otp-form - Conditional", + "description": "Flow to determine if the auth-otp-form authenticator should be used or not.", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "auth-otp-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + } + ] + }, + { + "id": "cffbb5df-de0a-49ed-9136-296a877ab175", + "alias": "http challenge", + "description": "An authentication flow based on challenge-response HTTP Authentication Schemes", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "no-cookie-redirect", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "basic-auth", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "basic-auth-otp", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 30, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "auth-spnego", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 40, + "autheticatorFlow": false, + "userSetupAllowed": false + } + ] + }, + { + "id": "6ac5a9df-dacb-462c-9b12-207470e9fcbf", + "alias": "registration", + "description": "registration flow", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "registration-page-form", + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": true, + "flowAlias": "registration form", + "userSetupAllowed": false + } + ] + }, + { + "id": "27e40f78-ce1e-4ad4-9b48-88a8bf9c8d92", + "alias": "registration form", + "description": "registration form", + "providerId": "form-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "registration-user-creation", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "registration-profile-action", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 40, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "registration-password-action", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 50, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "registration-recaptcha-action", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 60, + "autheticatorFlow": false, + "userSetupAllowed": false + } + ] + }, + { + "id": "31340e3b-f6c7-49ce-94ac-f28213b84be6", + "alias": "reset credentials", + "description": "Reset credentials for a user if they forgot their password or something", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "reset-credentials-choose-user", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "reset-credential-email", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "reset-password", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 30, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 40, + "autheticatorFlow": true, + "flowAlias": "reset credentials - reset-otp - Conditional", + "userSetupAllowed": false + } + ] + }, + { + "id": "aee4a6d9-caab-463e-ad62-48aba91a4098", + "alias": "reset credentials - reset-otp - Conditional", + "description": "Flow to determine if the reset-otp authenticator should be used or not.", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "reset-otp", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + } + ] + }, + { + "id": "4052bdf6-9b94-42a1-b199-0c14ffe67ac5", + "alias": "saml ecp", + "description": "SAML ECP Profile Authentication Flow", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "http-basic-authenticator", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + } + ] + } + ], + "authenticatorConfig": [ + { + "id": "4bc95f52-8c28-449c-830b-a4ffc3340399", + "alias": "create unique user config", + "config": { + "require.password.update.after.registration": "false" + } + }, + { + "id": "367a56fc-c128-43f8-85d5-50ceae63b7aa", + "alias": "review profile config", + "config": { + "update.profile.on.first.login": "missing" + } + } + ], + "requiredActions": [ + { + "alias": "CONFIGURE_TOTP", + "name": "Configure OTP", + "providerId": "CONFIGURE_TOTP", + "enabled": true, + "defaultAction": false, + "priority": 10, + "config": {} + }, + { + "alias": "terms_and_conditions", + "name": "Terms and Conditions", + "providerId": "terms_and_conditions", + "enabled": false, + "defaultAction": false, + "priority": 20, + "config": {} + }, + { + "alias": "UPDATE_PASSWORD", + "name": "Update Password", + "providerId": "UPDATE_PASSWORD", + "enabled": true, + "defaultAction": false, + "priority": 30, + "config": {} + }, + { + "alias": "UPDATE_PROFILE", + "name": "Update Profile", + "providerId": "UPDATE_PROFILE", + "enabled": true, + "defaultAction": false, + "priority": 40, + "config": {} + }, + { + "alias": "VERIFY_EMAIL", + "name": "Verify Email", + "providerId": "VERIFY_EMAIL", + "enabled": true, + "defaultAction": false, + "priority": 50, + "config": {} + }, + { + "alias": "delete_account", + "name": "Delete Account", + "providerId": "delete_account", + "enabled": false, + "defaultAction": false, + "priority": 60, + "config": {} + }, + { + "alias": "update_user_locale", + "name": "Update User Locale", + "providerId": "update_user_locale", + "enabled": true, + "defaultAction": false, + "priority": 1000, + "config": {} + } + ], + "browserFlow": "browser plus basic", + "registrationFlow": "registration", + "directGrantFlow": "direct grant", + "resetCredentialsFlow": "reset credentials", + "clientAuthenticationFlow": "clients", + "dockerAuthenticationFlow": "docker auth", + "attributes": { + "cibaBackchannelTokenDeliveryMode": "poll", + "cibaExpiresIn": "120", + "cibaAuthRequestedUserHint": "login_hint", + "oauth2DeviceCodeLifespan": "600", + "clientOfflineSessionMaxLifespan": "0", + "oauth2DevicePollingInterval": "5", + "clientSessionIdleTimeout": "0", + "parRequestUriLifespan": "60", + "clientSessionMaxLifespan": "0", + "clientOfflineSessionIdleTimeout": "0", + "cibaInterval": "5", + "realmReusableOtpCode": "false" + }, + "keycloakVersion": "20.0.2", + "userManagedAccessAllowed": false, + "clientProfiles": { + "profiles": [] + }, + "clientPolicies": { + "policies": [] + } +} \ No newline at end of file diff --git a/app/data/config/mainnet-eth-keycloak/keycloak.env b/app/data/config/mainnet-eth-keycloak/keycloak.env new file mode 100644 index 00000000..0bc7bf15 --- /dev/null +++ b/app/data/config/mainnet-eth-keycloak/keycloak.env @@ -0,0 +1,17 @@ +POSTGRES_DB=keycloak +POSTGRES_USER=keycloak +POSTGRES_PASSWORD=keycloak +KC_DB=postgres +KC_DB_URL_HOST=keycloak-db +KC_DB_URL_DATABASE=${POSTGRES_DB} +KC_DB_USERNAME=${POSTGRES_USER} +KC_DB_PASSWORD=${POSTGRES_PASSWORD} +KC_DB_SCHEMA=public +KC_HOSTNAME=localhost +KC_HTTP_ENABLED="true" +KC_HTTP_RELATIVE_PATH="/auth" +KC_HOSTNAME_STRICT_HTTPS="false" +KEYCLOAK_ADMIN=admin +KEYCLOAK_ADMIN_PASSWORD=admin +X_API_CHECK_REALM=cerc +X_API_CHECK_CLIENT_ID="%api_key%" diff --git a/app/data/config/mainnet-eth-keycloak/nginx/keycloak_proxy.conf b/app/data/config/mainnet-eth-keycloak/nginx/keycloak_proxy.conf new file mode 100644 index 00000000..dd69680b --- /dev/null +++ b/app/data/config/mainnet-eth-keycloak/nginx/keycloak_proxy.conf @@ -0,0 +1,72 @@ +server { + listen 80; + listen [::]:80; + server_name localhost; + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + +### geth + location ~ ^/eth/?([^/]*)$ { + set $apiKey $1; + if ($apiKey = '') { + set $apiKey $http_X_API_KEY; + } + auth_request /auth; + proxy_buffering off; + rewrite /.*$ / break; + proxy_pass http://fixturenet-eth-geth-1:8545; + } + +## ipld-eth-server +# location ~ ^/ipld/eth/([^/]*)$ { +# set $apiKey $1; +# if ($apiKey = '') { +# set $apiKey $http_X_API_KEY; +# } +# auth_request /auth; +# auth_request_set $user_id $sent_http_x_user_id; +# proxy_buffering off; +# rewrite /.*$ / break; +# proxy_pass http://ipld-eth-server:8081; +# proxy_set_header X-Original-Remote-Addr $remote_addr; +# proxy_set_header X-User-Id $user_id; +# } +# +# location ~ ^/ipld/gql/([^/]*)$ { +# set $apiKey $1; +# if ($apiKey = '') { +# set $apiKey $http_X_API_KEY; +# } +# auth_request /auth; +# proxy_buffering off; +# rewrite /.*$ / break; +# proxy_pass http://ipld-eth-server:8082; +# } + +## lighthouse + location /beacon/ { + set $apiKey $http_X_API_KEY; + auth_request /auth; + proxy_buffering off; + proxy_pass http://fixturenet-eth-lighthouse-1:8001/; + } + + location = /auth { + internal; + proxy_buffering off; + resolver 127.0.0.11 ipv6=off; + proxy_pass http://keycloak:8080/auth/realms/cerc/check?apiKey=$apiKey; + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-URI $request_uri; + proxy_set_header X-Original-Remote-Addr $remote_addr; + proxy_set_header X-Original-Host $host; + } + + location = /stub_status { + stub_status; + } +} diff --git a/app/data/config/mainnet-eth-metrics/grafana/etc/dashboards/eth_dashboard.json b/app/data/config/mainnet-eth-metrics/grafana/etc/dashboards/eth_dashboard.json new file mode 100644 index 00000000..1946ed37 --- /dev/null +++ b/app/data/config/mainnet-eth-metrics/grafana/etc/dashboards/eth_dashboard.json @@ -0,0 +1,1409 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 16, + "panels": [], + "title": "Row title", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(nginx_http_requests_total{instance=\"keycloak-nginx-prometheus-exporter:9113\"}[$__rate_interval])", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "nginx Requests/sec", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "builder", + "expr": "rate(ipld_eth_server_http_count{instance=\"ipld-eth-server:8090\"}[$__rate_interval])", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "ipld-eth-server Requests/sec", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "Time" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "{instance=\"ipld-eth-server:8090\", job=\"ipld-eth-server\"}": true + }, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_blockNumber_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_blockNumber", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_call_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_call", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_chainId_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_chainId", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_estimateGas_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_estimateGas", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_gasPrice_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_gasPrice", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getBalance_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getBalance", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getBlockByHash_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getBlockByHash", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getBlockByNumber_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getBlockByNumber", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getBlockReceipts_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getBlockReceipts", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getBlockTransactionCountByHash_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getBlockTransactionCountByHash", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getBlockTransactionCountByNumber_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getBlockTransactionCountByNumber", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getCode_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getCode", + "range": true, + "refId": "L" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getFilterChanges_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getFilterChanges", + "range": true, + "refId": "M" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getFilterLogs_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getFilterLogs", + "range": true, + "refId": "N" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getLogs_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getLogs", + "range": true, + "refId": "O" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getStorageAt_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getStorageAt", + "range": true, + "refId": "P" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getTransactionByBlockHashAndIndex_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getTransactionByBlockHashAndIndex", + "range": true, + "refId": "Q" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getTransactionByBlockNumberAndIndex_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getTransactionByBlockNumberAndIndex", + "range": true, + "refId": "R" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getTransactionByHash_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getTransactionByHash", + "range": true, + "refId": "S" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getTransactionCount_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getTransactionCount", + "range": true, + "refId": "T" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getTransactionReceipt_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getTransactionReceipt", + "range": true, + "refId": "U" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getUncleCountByBlockHash_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getUncleCountByBlockHash", + "range": true, + "refId": "V" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getUncleCountByBlockNumber_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getUncleCountByBlockNumber", + "range": true, + "refId": "W" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_mining_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_mining_success", + "range": true, + "refId": "X" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_newBlockFilter_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_newBlockFilter", + "range": true, + "refId": "Y" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_newFilter_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_newFilter", + "range": true, + "refId": "Z" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_newPendingTransactionFilter_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_newPendingTransactionFilter", + "range": true, + "refId": "AA" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_sendRawTransaction_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_sendRawTransaction", + "range": true, + "refId": "AB" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_signTransaction_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_signTransaction", + "range": true, + "refId": "AC" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_subscribe_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_subscribe", + "range": true, + "refId": "AD" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_syncing_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_syncing", + "range": true, + "refId": "AE" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_unsubscribe_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_unsubscribe", + "range": true, + "refId": "AF" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_statediff_writeStateDiffFor_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "statediff_writeStateDiffFor", + "range": true, + "refId": "AG" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_statediff_writeStateDiffAt_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "statediff_writeStateDiffAt", + "range": true, + "refId": "AH" + } + ], + "title": "geth API Requests/sec", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(keycloak_logins{instance=\"keycloak:8080\"}[90s]) * 60", + "format": "time_series", + "instant": false, + "interval": "2m", + "legendFormat": "{{client_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Keycloak Logins/min", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "Time" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "00c13eee-e48c-4c7f-872b-f0e4c1e67d7f": true, + "0cf1b92d-a15c-40a8-b2ae-49a758e658ee": true, + "13761bf2-2c0e-42fd-acc8-09a677ab179e": true, + "79a56c4e-0bfb-4ef0-87d4-244c4ace9b4b": true, + "9c41e289-92f0-456f-bd65-8f73024a9e45": true, + "c9718dd1-06d6-4f98-a821-9adbeb4b663b": true + }, + "indexByName": {}, + "renameByName": { + "00c13eee-e48c-4c7f-872b-f0e4c1e67d7f": "", + "0cf1b92d-a15c-40a8-b2ae-49a758e658ee": "rickm", + "13761bf2-2c0e-42fd-acc8-09a677ab179e": "telackey", + "2753f8be-0036-49ba-a53a-4963573fc15c": "cerc-io", + "79a56c4e-0bfb-4ef0-87d4-244c4ace9b4b": "", + "9c41e289-92f0-456f-bd65-8f73024a9e45": "", + "c9718dd1-06d6-4f98-a821-9adbeb4b663b": "dboreham", + "f5083aa7-5b5d-4164-b189-d7a559c4aad0": "infura0" + } + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 17 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "chain_head_block{instance=\"${node}:6060\"}", + "hide": false, + "legendFormat": "geth", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "statediff_service_last_statediff_height{instance=\"${node}:6060\"}", + "legendFormat": "statediff", + "range": true, + "refId": "A" + } + ], + "title": "Block Positions", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Distance" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 17 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "clamp_min(chain_head_block{instance=\"${node}:6060\"} - on() statediff_service_last_statediff_height{instance=\"${node}:6060\"}, 0)", + "interval": "1m", + "legendFormat": "Distance", + "range": true, + "refId": "A" + } + ], + "title": "Block Distance (geth to statediff)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(statediff_service_succeeded{instance=\"${node}:6060\"}[1m])", + "format": "time_series", + "instant": false, + "interval": "1m", + "legendFormat": "Succeeded", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "increase(statediff_service_failed{instance=\"${node}:6060\"}[1m])", + "hide": false, + "interval": "1m", + "legendFormat": "Failed", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "statediff_service_underway{instance=\"${node}:6060\"}", + "hide": false, + "interval": "1s", + "legendFormat": "Underway", + "range": true, + "refId": "C" + } + ], + "title": "Statediff Operations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 50 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(statediff_service_total_processing_time{instance=\"${node}:6060\"}[$__rate_interval])", + "instant": false, + "interval": "1m", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Statediff Duty Cycle", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(statediff_service_api_requests{instance=\"${node}:6060\"}[$__rate_interval])", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Statediff API Requests/sec", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "mainnet-eth-geth-1", + "value": "mainnet-eth-geth-1" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "node", + "options": [ + { + "selected": true, + "text": "mainnet-eth-geth-1", + "value": "mainnet-eth-geth-1" + } + ], + "query": "mainnet-eth-geth-1", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "cerc-io overview", + "uid": "nT9VeZoVk", + "version": 2, + "weekStart": "" +} diff --git a/app/data/config/mainnet-eth-metrics/grafana/etc/provisioning/dashboards/dashboards.yml b/app/data/config/mainnet-eth-metrics/grafana/etc/provisioning/dashboards/dashboards.yml new file mode 100644 index 00000000..4705688e --- /dev/null +++ b/app/data/config/mainnet-eth-metrics/grafana/etc/provisioning/dashboards/dashboards.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +providers: + - name: dashboards + type: file + updateIntervalSeconds: 10 + options: + path: /etc/grafana/dashboards + foldersFromFilesStructure: true diff --git a/app/data/config/mainnet-eth-metrics/grafana/etc/provisioning/datasources/prometheus.yml b/app/data/config/mainnet-eth-metrics/grafana/etc/provisioning/datasources/prometheus.yml new file mode 100644 index 00000000..f5181ebd --- /dev/null +++ b/app/data/config/mainnet-eth-metrics/grafana/etc/provisioning/datasources/prometheus.yml @@ -0,0 +1,19 @@ +apiVersion: 1 + +datasources: + - id: 1 + uid: jZUuGao4k + orgId: 1 + name: Prometheus + type: prometheus + typeName: Prometheus + typeLogoUrl: public/app/plugins/datasource/prometheus/img/prometheus_logo.svg + access: proxy + url: http://prometheus:9090 + user: "" + database: "" + basicAuth: false + isDefault: true + jsonData: + httpMethod: POST + readOnly: false diff --git a/app/data/config/mainnet-eth-metrics/prometheus/etc/prometheus.yml b/app/data/config/mainnet-eth-metrics/prometheus/etc/prometheus.yml new file mode 100644 index 00000000..e9f23e89 --- /dev/null +++ b/app/data/config/mainnet-eth-metrics/prometheus/etc/prometheus.yml @@ -0,0 +1,19 @@ +global: + scrape_interval: 5s + evaluation_interval: 15s + +scrape_configs: + # geth + - job_name: 'geth' + metrics_path: /debug/metrics/prometheus + scheme: http + static_configs: + - targets: ['mainnet-eth-geth-1:6060'] + + # keycloak + - job_name: 'keycloak' + scrape_interval: 5s + metrics_path: /auth/realms/cerc/metrics + scheme: http + static_configs: + - targets: ['keycloak:8080'] diff --git a/app/data/config/mainnet-eth/geth.env b/app/data/config/mainnet-eth/geth.env new file mode 100644 index 00000000..37b97d02 --- /dev/null +++ b/app/data/config/mainnet-eth/geth.env @@ -0,0 +1,57 @@ +# Enable remote debugging using dlv +CERC_REMOTE_DEBUG=false + +# Enable startup script debug output. +CERC_SCRIPT_DEBUG=false + +# Simple toggle to choose either a 'full' node or an 'archive' node +# (controls the values of --syncmode --gcmode --snapshot) +CERC_GETH_MODE_QUICK_SET=archive + +# Optional custom node name. +# GETH_NODE_NAME="" + +# Specify any other geth CLI options. +GETH_OPTS="" + +# --cache +GETH_CACHE=1024 + +# --cache.database +GETH_CACHE_DB=50 + +# --cache.gc +GETH_CACHE_GC=25 + +# --cache.trie +GETH_CACHE_TRIE=15 +j +# --datadir +GETH_DATADIR="/data" + +# --http.api +GETH_HTTP_API="eth,web3,net" + +# --authrpc.jwtsecret +GETH_JWTSECRET="/etc/mainnet-eth/jwtsecret" + +# --maxpeers +GETH_MAX_PEERS=100 + +# --rpc.evmtimeout +GETH_RPC_EVMTIMEOUT=0 + +# --rpc.gascap +GETH_RPC_GASCAP=0 + +# --txlookuplimit +GETH_TXLOOKUPLIMIT=0 + +# --verbosity +GETH_VERBOSITY=3 + +# --log.vmodule +GETH_VMODULE="rpc/*=4" + +# --ws.api +GETH_WS_API="eth,web3,net" diff --git a/app/data/config/mainnet-eth/lighthouse.env b/app/data/config/mainnet-eth/lighthouse.env new file mode 100644 index 00000000..1fd5ecdb --- /dev/null +++ b/app/data/config/mainnet-eth/lighthouse.env @@ -0,0 +1,30 @@ +# Enable remote debugging using dlv +CERC_REMOTE_DEBUG=false + +# Enable startup script debug output. +CERC_SCRIPT_DEBUG=false + +# Specify any other lighthouse CLI options. +LIGHTHOUSE_OPTS="" + +# Override the advertised public IP (optional) +# --enr-address +#LIGHTHOUSE_ENR_ADDRESS="" + +# --checkpoint-sync-url +LIGHTHOUSE_CHECKPOINT_SYNC_URL=https://beaconstate.ethstaker.cc + +# --datadir +LIGHTHOUSE_DATADIR=/data + +# --debug-level +LIGHTHOUSE_DEBUG_LEVEL=info + +# --http-port +LIGHTHOUSE_HTTP_PORT=5052 + +# --execution-jwt +LIGHTHOUSE_JWTSECRET=/etc/mainnet-eth/jwtsecret + +# --port --enr-udp-port --enr-tcp-port +LIGHTHOUSE_NETWORK_PORT=9000 diff --git a/app/data/config/mainnet-eth/scripts/run-geth.sh b/app/data/config/mainnet-eth/scripts/run-geth.sh index 3844ef99..dafa77ae 100755 --- a/app/data/config/mainnet-eth/scripts/run-geth.sh +++ b/app/data/config/mainnet-eth/scripts/run-geth.sh @@ -3,10 +3,8 @@ if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then set -x fi -CERC_ETH_DATADIR=/root/ethdata - START_CMD="geth" -if [ "true" == "$CERC_REMOTE_DEBUG" ] && [ -x "/usr/local/bin/dlv" ]; then +if [[ "true" == "$CERC_REMOTE_DEBUG" ]] && [[ -x "/usr/local/bin/dlv" ]]; then START_CMD="/usr/local/bin/dlv --listen=:40000 --headless=true --api-version=2 --accept-multiclient exec /usr/local/bin/geth --continue --" fi @@ -22,29 +20,44 @@ cleanup() { wait echo "Done" } - trap 'cleanup' SIGINT SIGTERM +MODE_FLAGS="" +if [[ "$CERC_GETH_MODE_QUICK_SET" = "archive" ]]; then + MODE_FLAGS="--syncmode=${GETH_SYNC_MODE:-full} --gcmode=${GETH_GC_MODE:-archive} --snapshot=${GETH_SNAPSHOT:-false}" +else + MODE_FLAGS="--syncmode=${GETH_SYNC_MODE:-snap} --gcmode=${GETH_GC_MODE:-snap} --snapshot=${GETH_SNAPSHOT:-true}" +fi + $START_CMD \ - --datadir="${CERC_ETH_DATADIR}" \ - --authrpc.addr="0.0.0.0" \ - --authrpc.port 8551 \ - --authrpc.vhosts="*" \ - --authrpc.jwtsecret="/etc/mainnet-eth/jwtsecret" \ - --ws \ - --ws.addr="0.0.0.0" \ - --ws.origins="*" \ - --ws.api="${CERC_GETH_WS_APIS:-eth,web3,net,admin,personal,debug,statediff}" \ - --http.corsdomain="*" \ - --gcmode full \ - --txlookuplimit=0 \ - --cache.preimages \ - --syncmode=snap \ - & + $MODE_FLAGS \ + --datadir="${GETH_DATA}"\ + --identity="${GETH_NODE_NAME}" \ + --maxpeers=${GETH_MAX_PEERS} \ + --cache=${GETH_CACHE} \ + --cache.gc=${GETH_CACHE_GC} \ + --cache.database=${GETH_CACHE_DB} \ + --cache.trie=${GETH_CACHE_TRIE} + --authrpc.addr='0.0.0.0' \ + --authrpc.vhosts='*' \ + --authrpc.jwtsecret="${GETH_JWTSECRET}" \ + --http \ + --http.addr='0.0.0.0' \ + --http.api="${GETH_HTTP_API}" \ + --http.vhosts='*' \ + --metrics \ + --metrics.addr='0.0.0.0' \ + --ws \ + --ws.addr='0.0.0.0' \ + --ws.api="${GETH_WS_API}" \ + --rpc.gascap=${GETH_RPC_GASCAP} \ + --rpc.evmtimeout=${GETH_RPC_EVMTIMEOUT} \ + --txlookuplimit=${GETH_TXLOOKUPLIMIT} + --verbosity=${GETH_VERBOSITY} \ + --log.vmodule="${GETH_VMODULE}" \ + ${GETH_OPTS} & geth_pid=$! - - wait $geth_pid if [ "true" == "$CERC_KEEP_RUNNING_AFTER_GETH_EXIT" ]; then diff --git a/app/data/config/mainnet-eth/scripts/run-lighthouse.sh b/app/data/config/mainnet-eth/scripts/run-lighthouse.sh index 02c97922..3a079410 100755 --- a/app/data/config/mainnet-eth/scripts/run-lighthouse.sh +++ b/app/data/config/mainnet-eth/scripts/run-lighthouse.sh @@ -3,20 +3,22 @@ if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then set -x fi -DEBUG_LEVEL=${CERC_LIGHTHOUSE_DEBUG_LEVEL:-info} +ENR_OPTS="" +if [[ -n "$LIGHTHOUSE_ENR_ADDRESS" ]]; then + ENR_OPTS="--enr-address $LIGHTHOUSE_ENR_ADDRESS" +fi -data_dir=/var/lighthouse-data-dir - -network_port=9001 -http_port=8001 -authrpc_port=8551 - -exec lighthouse \ - bn \ - --debug-level $DEBUG_LEVEL \ - --datadir $data_dir \ - --network mainnet \ - --execution-endpoint $EXECUTION_ENDPOINT \ - --execution-jwt /etc/mainnet-eth/jwtsecret \ +exec lighthouse bn \ + --checkpoint-sync-url "$LIGHTHOUSE_CHECKPOINT_SYNC_URL" \ + --datadir "$LIGHTHOUSE_DATADIR" \ + --debug-level $LIGHTHOUSE_DEBUG_LEVEL \ --disable-deposit-contract-sync \ - --checkpoint-sync-url https://beaconstate.ethstaker.cc + --enr-tcp-port $LIGHTHOUSE_NETWORK_PORT \ + --enr-udp-port $LIGHTHOUSE_NETWORK_PORT \ + --execution-endpoint "$EXECUTION_ENDPOINT" \ + --execution-jwt /etc/mainnet-eth/jwtsecret \ + --http-address 0.0.0.0 \ + --http-port $LIGHTHOUSE_HTTP_PORT \ + --network mainnet \ + --port $LIGHTHOUSE_NETWORK_PORT \ + $ENR_OPTS $LIGHTHOUSE_OPTS \ No newline at end of file diff --git a/app/data/stacks/mainnet-eth/stack.yml b/app/data/stacks/mainnet-eth/stack.yml index 5051eb9a..11885cef 100644 --- a/app/data/stacks/mainnet-eth/stack.yml +++ b/app/data/stacks/mainnet-eth/stack.yml @@ -10,6 +10,9 @@ containers: - cerc/lighthouse - cerc/lighthouse-cli - cerc/foundry + - cerc/keycloak pods: - mainnet-eth + - mainnet-eth-keycloak + - mainnet-eth-metrics - foundry -- 2.45.2 From ed4abe9aea4ea6726942cdf39ec3d77d3b091220 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Fri, 11 Aug 2023 15:42:06 -0500 Subject: [PATCH 03/16] EOL --- app/data/compose/docker-compose-mainnet-eth-keycloak.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/data/compose/docker-compose-mainnet-eth-keycloak.yml b/app/data/compose/docker-compose-mainnet-eth-keycloak.yml index c4ebdea8..89447bf8 100644 --- a/app/data/compose/docker-compose-mainnet-eth-keycloak.yml +++ b/app/data/compose/docker-compose-mainnet-eth-keycloak.yml @@ -32,4 +32,4 @@ services: condition: service_healthy volumes: - mainnet_eth_keycloak_db: \ No newline at end of file + mainnet_eth_keycloak_db: -- 2.45.2 From 010294f1022a37137dbc15f5e1280d12cf148737 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Fri, 11 Aug 2023 15:43:05 -0500 Subject: [PATCH 04/16] EOL --- app/data/config/mainnet-eth/scripts/run-lighthouse.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/data/config/mainnet-eth/scripts/run-lighthouse.sh b/app/data/config/mainnet-eth/scripts/run-lighthouse.sh index 3a079410..5f0c66e2 100755 --- a/app/data/config/mainnet-eth/scripts/run-lighthouse.sh +++ b/app/data/config/mainnet-eth/scripts/run-lighthouse.sh @@ -21,4 +21,4 @@ exec lighthouse bn \ --http-port $LIGHTHOUSE_HTTP_PORT \ --network mainnet \ --port $LIGHTHOUSE_NETWORK_PORT \ - $ENR_OPTS $LIGHTHOUSE_OPTS \ No newline at end of file + $ENR_OPTS $LIGHTHOUSE_OPTS -- 2.45.2 From bbdceda5b0c7eedde26ceeb120c82b9fe944f88c Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Sat, 12 Aug 2023 00:10:28 +0000 Subject: [PATCH 05/16] Missing ports. --- app/data/compose/docker-compose-mainnet-eth.yml | 14 ++++++++++++++ app/data/config/mainnet-eth/lighthouse.env | 3 +++ .../config/mainnet-eth/scripts/run-lighthouse.sh | 5 +++++ 3 files changed, 22 insertions(+) diff --git a/app/data/compose/docker-compose-mainnet-eth.yml b/app/data/compose/docker-compose-mainnet-eth.yml index 3a76b593..04f5b236 100644 --- a/app/data/compose/docker-compose-mainnet-eth.yml +++ b/app/data/compose/docker-compose-mainnet-eth.yml @@ -22,9 +22,18 @@ services: retries: 10 start_period: 3s ports: + # http api - "8545" + # ws api + - "8546" + # ws el - "8551" + # p2p + - "30303" + - "30303/udp" + # debugging - "40000" + # metrics - "6060" mainnet-eth-lighthouse-1: @@ -48,8 +57,13 @@ services: - mainnet_eth_config_data:/etc/mainnet-eth - ../config/mainnet-eth/scripts/run-lighthouse.sh:/opt/run-lighthouse.sh ports: + # api - "5052" + # metrics + - "5054" + # p2p - "9000" + - "9000/udp" volumes: mainnet_eth_config_data: diff --git a/app/data/config/mainnet-eth/lighthouse.env b/app/data/config/mainnet-eth/lighthouse.env index 1fd5ecdb..7a07b987 100644 --- a/app/data/config/mainnet-eth/lighthouse.env +++ b/app/data/config/mainnet-eth/lighthouse.env @@ -26,5 +26,8 @@ LIGHTHOUSE_HTTP_PORT=5052 # --execution-jwt LIGHTHOUSE_JWTSECRET=/etc/mainnet-eth/jwtsecret +# --metrics-port +LIGHTHOUSE_METRICS_PORT=5054 + # --port --enr-udp-port --enr-tcp-port LIGHTHOUSE_NETWORK_PORT=9000 diff --git a/app/data/config/mainnet-eth/scripts/run-lighthouse.sh b/app/data/config/mainnet-eth/scripts/run-lighthouse.sh index 5f0c66e2..af98771b 100755 --- a/app/data/config/mainnet-eth/scripts/run-lighthouse.sh +++ b/app/data/config/mainnet-eth/scripts/run-lighthouse.sh @@ -13,12 +13,17 @@ exec lighthouse bn \ --datadir "$LIGHTHOUSE_DATADIR" \ --debug-level $LIGHTHOUSE_DEBUG_LEVEL \ --disable-deposit-contract-sync \ + --disable-upnp \ --enr-tcp-port $LIGHTHOUSE_NETWORK_PORT \ --enr-udp-port $LIGHTHOUSE_NETWORK_PORT \ --execution-endpoint "$EXECUTION_ENDPOINT" \ --execution-jwt /etc/mainnet-eth/jwtsecret \ + --http \ --http-address 0.0.0.0 \ --http-port $LIGHTHOUSE_HTTP_PORT \ + --metrics \ + --metrics-address=0.0.0.0 \ + --metrics-port $LIGHTHOUSE_METRICS_PORT \ --network mainnet \ --port $LIGHTHOUSE_NETWORK_PORT \ $ENR_OPTS $LIGHTHOUSE_OPTS -- 2.45.2 From 1a012b522d9f0afd20a5e7af45f0d48ff3eefe7c Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Sat, 12 Aug 2023 00:52:48 +0000 Subject: [PATCH 06/16] Minor script fixes. --- app/data/config/mainnet-eth/lighthouse.env | 8 ++++---- app/data/config/mainnet-eth/scripts/run-geth.sh | 4 ++-- app/data/config/mainnet-eth/scripts/run-lighthouse.sh | 5 +++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/app/data/config/mainnet-eth/lighthouse.env b/app/data/config/mainnet-eth/lighthouse.env index 7a07b987..11fc6b69 100644 --- a/app/data/config/mainnet-eth/lighthouse.env +++ b/app/data/config/mainnet-eth/lighthouse.env @@ -1,6 +1,3 @@ -# Enable remote debugging using dlv -CERC_REMOTE_DEBUG=false - # Enable startup script debug output. CERC_SCRIPT_DEBUG=false @@ -12,7 +9,10 @@ LIGHTHOUSE_OPTS="" #LIGHTHOUSE_ENR_ADDRESS="" # --checkpoint-sync-url -LIGHTHOUSE_CHECKPOINT_SYNC_URL=https://beaconstate.ethstaker.cc +LIGHTHOUSE_CHECKPOINT_SYNC_URL="https://beaconstate.ethstaker.cc" + +# --checkpoint-sync-url-timeout +LIGHTHOUSE_CHECKPOINT_SYNC_URL_TIMEOUT=300 # --datadir LIGHTHOUSE_DATADIR=/data diff --git a/app/data/config/mainnet-eth/scripts/run-geth.sh b/app/data/config/mainnet-eth/scripts/run-geth.sh index dafa77ae..5e0b6786 100755 --- a/app/data/config/mainnet-eth/scripts/run-geth.sh +++ b/app/data/config/mainnet-eth/scripts/run-geth.sh @@ -1,5 +1,5 @@ #!/bin/sh -if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then +if [[ "true" == "$CERC_SCRIPT_DEBUG" ]]; then set -x fi @@ -37,7 +37,7 @@ $START_CMD \ --cache=${GETH_CACHE} \ --cache.gc=${GETH_CACHE_GC} \ --cache.database=${GETH_CACHE_DB} \ - --cache.trie=${GETH_CACHE_TRIE} + --cache.trie=${GETH_CACHE_TRIE} \ --authrpc.addr='0.0.0.0' \ --authrpc.vhosts='*' \ --authrpc.jwtsecret="${GETH_JWTSECRET}" \ diff --git a/app/data/config/mainnet-eth/scripts/run-lighthouse.sh b/app/data/config/mainnet-eth/scripts/run-lighthouse.sh index af98771b..efda735b 100755 --- a/app/data/config/mainnet-eth/scripts/run-lighthouse.sh +++ b/app/data/config/mainnet-eth/scripts/run-lighthouse.sh @@ -1,5 +1,5 @@ #!/bin/bash -if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then +if [[ "true" == "$CERC_SCRIPT_DEBUG" ]]; then set -x fi @@ -10,13 +10,14 @@ fi exec lighthouse bn \ --checkpoint-sync-url "$LIGHTHOUSE_CHECKPOINT_SYNC_URL" \ + --checkpoint-sync-url-timeout ${LIGHTHOUSE_CHECKPOINT_SYNC_URL_TIMEOUT} \ --datadir "$LIGHTHOUSE_DATADIR" \ --debug-level $LIGHTHOUSE_DEBUG_LEVEL \ --disable-deposit-contract-sync \ --disable-upnp \ --enr-tcp-port $LIGHTHOUSE_NETWORK_PORT \ --enr-udp-port $LIGHTHOUSE_NETWORK_PORT \ - --execution-endpoint "$EXECUTION_ENDPOINT" \ + --execution-endpoint "$LIGHTHOUSE_EXECUTION_ENDPOINT" \ --execution-jwt /etc/mainnet-eth/jwtsecret \ --http \ --http-address 0.0.0.0 \ -- 2.45.2 From e61bdc8f72aee46f66b8b11999019bd594ee7f5a Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Sat, 12 Aug 2023 00:59:43 +0000 Subject: [PATCH 07/16] Minor script fixes. --- app/data/config/mainnet-eth/geth.env | 2 +- app/data/config/mainnet-eth/scripts/run-geth.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/data/config/mainnet-eth/geth.env b/app/data/config/mainnet-eth/geth.env index 37b97d02..a01444df 100644 --- a/app/data/config/mainnet-eth/geth.env +++ b/app/data/config/mainnet-eth/geth.env @@ -6,7 +6,7 @@ CERC_SCRIPT_DEBUG=false # Simple toggle to choose either a 'full' node or an 'archive' node # (controls the values of --syncmode --gcmode --snapshot) -CERC_GETH_MODE_QUICK_SET=archive +CERC_GETH_MODE_QUICK_SET=full # Optional custom node name. # GETH_NODE_NAME="" diff --git a/app/data/config/mainnet-eth/scripts/run-geth.sh b/app/data/config/mainnet-eth/scripts/run-geth.sh index 5e0b6786..d6c6afb6 100755 --- a/app/data/config/mainnet-eth/scripts/run-geth.sh +++ b/app/data/config/mainnet-eth/scripts/run-geth.sh @@ -26,7 +26,7 @@ MODE_FLAGS="" if [[ "$CERC_GETH_MODE_QUICK_SET" = "archive" ]]; then MODE_FLAGS="--syncmode=${GETH_SYNC_MODE:-full} --gcmode=${GETH_GC_MODE:-archive} --snapshot=${GETH_SNAPSHOT:-false}" else - MODE_FLAGS="--syncmode=${GETH_SYNC_MODE:-snap} --gcmode=${GETH_GC_MODE:-snap} --snapshot=${GETH_SNAPSHOT:-true}" + MODE_FLAGS="--syncmode=${GETH_SYNC_MODE:-snap} --gcmode=${GETH_GC_MODE:-full} --snapshot=${GETH_SNAPSHOT:-true}" fi $START_CMD \ -- 2.45.2 From bc4cacea87e0299e78e9e7f83a9c2c1630237078 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Sat, 12 Aug 2023 01:32:15 +0000 Subject: [PATCH 08/16] Minor script fixes. --- app/data/config/mainnet-eth/scripts/run-geth.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/data/config/mainnet-eth/scripts/run-geth.sh b/app/data/config/mainnet-eth/scripts/run-geth.sh index d6c6afb6..85730aec 100755 --- a/app/data/config/mainnet-eth/scripts/run-geth.sh +++ b/app/data/config/mainnet-eth/scripts/run-geth.sh @@ -52,7 +52,7 @@ $START_CMD \ --ws.api="${GETH_WS_API}" \ --rpc.gascap=${GETH_RPC_GASCAP} \ --rpc.evmtimeout=${GETH_RPC_EVMTIMEOUT} \ - --txlookuplimit=${GETH_TXLOOKUPLIMIT} + --txlookuplimit=${GETH_TXLOOKUPLIMIT} \ --verbosity=${GETH_VERBOSITY} \ --log.vmodule="${GETH_VMODULE}" \ ${GETH_OPTS} & -- 2.45.2 From 5cee6b8778e1df99939cbc1dfecd89a0c80052be Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Fri, 11 Aug 2023 22:29:07 -0500 Subject: [PATCH 09/16] Add keycloak-mirror script. --- .../nginx/keycloak_proxy.conf | 72 --------- .../keycloak-mirror/keycloak-mirror.py | 138 ++++++++++++++++++ .../scripts/keycloak-mirror/requirements.txt | 3 + 3 files changed, 141 insertions(+), 72 deletions(-) delete mode 100644 app/data/config/mainnet-eth-keycloak/nginx/keycloak_proxy.conf create mode 100755 app/data/config/mainnet-eth-keycloak/scripts/keycloak-mirror/keycloak-mirror.py create mode 100644 app/data/config/mainnet-eth-keycloak/scripts/keycloak-mirror/requirements.txt diff --git a/app/data/config/mainnet-eth-keycloak/nginx/keycloak_proxy.conf b/app/data/config/mainnet-eth-keycloak/nginx/keycloak_proxy.conf deleted file mode 100644 index dd69680b..00000000 --- a/app/data/config/mainnet-eth-keycloak/nginx/keycloak_proxy.conf +++ /dev/null @@ -1,72 +0,0 @@ -server { - listen 80; - listen [::]:80; - server_name localhost; - - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root /usr/share/nginx/html; - } - -### geth - location ~ ^/eth/?([^/]*)$ { - set $apiKey $1; - if ($apiKey = '') { - set $apiKey $http_X_API_KEY; - } - auth_request /auth; - proxy_buffering off; - rewrite /.*$ / break; - proxy_pass http://fixturenet-eth-geth-1:8545; - } - -## ipld-eth-server -# location ~ ^/ipld/eth/([^/]*)$ { -# set $apiKey $1; -# if ($apiKey = '') { -# set $apiKey $http_X_API_KEY; -# } -# auth_request /auth; -# auth_request_set $user_id $sent_http_x_user_id; -# proxy_buffering off; -# rewrite /.*$ / break; -# proxy_pass http://ipld-eth-server:8081; -# proxy_set_header X-Original-Remote-Addr $remote_addr; -# proxy_set_header X-User-Id $user_id; -# } -# -# location ~ ^/ipld/gql/([^/]*)$ { -# set $apiKey $1; -# if ($apiKey = '') { -# set $apiKey $http_X_API_KEY; -# } -# auth_request /auth; -# proxy_buffering off; -# rewrite /.*$ / break; -# proxy_pass http://ipld-eth-server:8082; -# } - -## lighthouse - location /beacon/ { - set $apiKey $http_X_API_KEY; - auth_request /auth; - proxy_buffering off; - proxy_pass http://fixturenet-eth-lighthouse-1:8001/; - } - - location = /auth { - internal; - proxy_buffering off; - resolver 127.0.0.11 ipv6=off; - proxy_pass http://keycloak:8080/auth/realms/cerc/check?apiKey=$apiKey; - proxy_pass_request_body off; - proxy_set_header Content-Length ""; - proxy_set_header X-Original-URI $request_uri; - proxy_set_header X-Original-Remote-Addr $remote_addr; - proxy_set_header X-Original-Host $host; - } - - location = /stub_status { - stub_status; - } -} diff --git a/app/data/config/mainnet-eth-keycloak/scripts/keycloak-mirror/keycloak-mirror.py b/app/data/config/mainnet-eth-keycloak/scripts/keycloak-mirror/keycloak-mirror.py new file mode 100755 index 00000000..5f553359 --- /dev/null +++ b/app/data/config/mainnet-eth-keycloak/scripts/keycloak-mirror/keycloak-mirror.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 + +import argparse +import os +import sys + +import psycopg +import random + +from subprocess import Popen +from fabric import Connection + + +def dump_src_db_to_file(db_host, db_port, db_user, db_password, db_name, file_name): + command = f"pg_dump -h {db_host} -p {db_port} -U {db_user} -d {db_name} -c --inserts -f {file_name}" + my_env = os.environ.copy() + my_env["PGPASSWORD"] = db_password + print(f"Exporting from {db_host}:{db_port}/{db_name} to {file_name}... ", end="") + ret = Popen(command, shell=True, env=my_env).wait() + print("DONE") + return ret + + +def establish_ssh_tunnel(ssh_host, ssh_port, ssh_user, db_host, db_port): + local_port = random.randint(11000, 12000) + conn = Connection(host=ssh_host, port=ssh_port, user=ssh_user) + fw = conn.forward_local( + local_port=local_port, remote_port=db_port, remote_host=db_host + ) + return conn, fw, local_port + + +def load_db_from_file(db_host, db_port, db_user, db_password, db_name, file_name): + connstr = "host=%s port=%s user=%s password=%s sslmode=disable dbname=%s" % ( + db_host, + db_port, + db_user, + db_password, + db_name, + ) + with psycopg.connect(connstr) as conn: + with conn.cursor() as cur: + print( + f"Importing from {file_name} to {db_host}:{db_port}/{db_name}... ", + end="", + ) + cur.execute(open(file_name, "rt").read()) + print("DONE") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument("--src-dbhost", help="DB hostname", default="localhost") + parser.add_argument("--src-dbport", help="DB port", default=5432, type=int) + parser.add_argument("--src-dbuser", help="DB username", default="keycloak") + parser.add_argument("--src-dbpw", help="DB password", required=True) + parser.add_argument("--src-dbname", help="dbname", default="keycloak") + + parser.add_argument( + "--dst-file", help="Destination filename", default="keycloak-mirror.sql" + ) + + parser.add_argument("--live-import", help="run the import", action="store_true") + + parser.add_argument("--dst-dbhost", help="DB hostname", default="localhost") + parser.add_argument("--dst-dbport", help="DB port", default=5432, type=int) + parser.add_argument("--dst-dbuser", help="DB username", default="keycloak") + parser.add_argument("--dst-dbpw", help="DB password") + parser.add_argument("--dst-dbname", help="dbname", default="keycloak") + + parser.add_argument("--ssh-host", help="SSH hostname") + parser.add_argument("--ssh-port", help="SSH port", default=22, type=int) + parser.add_argument("--ssh-user", help="SSH user") + + args = parser.parse_args() + + if args.live_import and not args.dst_dbpw: + print("--dst-dbpw is required if importing", file=sys.stderr) + sys.exit(2) + + remove_sql_file = False + if args.dst_dbhost and not args.dst_file: + remove_sql_file = True + + dst_file = args.dst_file + if not dst_file: + dst_file = "keycloak-mirror.sql" + + dump_src_db_to_file( + args.src_dbhost, + args.src_dbport, + args.src_dbuser, + args.src_dbpw, + args.src_dbname, + dst_file, + ) + + if args.live_import: + try: + if args.ssh_host: + dst_dbport = random.randint(11000, 12000) + print( + f"Establishing SSH tunnel from 127.0.0.1:{dst_dbport} to {args.ssh_host}->{args.dst_dbhost}:{args.dst_dbport}... ", + end="", + ) + with Connection( + host=args.ssh_host, port=args.ssh_port, user=args.ssh_user + ).forward_local( + local_port=dst_dbport, + remote_port=args.dst_dbport, + remote_host=args.dst_dbhost, + ): + print("DONE") + + load_db_from_file( + args.dst_dbhost, + args.dst_dbport, + args.dst_dbuser, + args.dst_dbpw, + args.dst_dbname, + dst_file, + ) + else: + load_db_from_file( + args.dst_dbhost, + args.dst_dbport, + args.dst_dbuser, + args.dst_dbpw, + args.dst_dbname, + dst_file, + ) + + finally: + if args.live_import: + print(f"Removing {dst_file}... ", end="") + os.remove(dst_file) + print("DONE") diff --git a/app/data/config/mainnet-eth-keycloak/scripts/keycloak-mirror/requirements.txt b/app/data/config/mainnet-eth-keycloak/scripts/keycloak-mirror/requirements.txt new file mode 100644 index 00000000..45b2bc57 --- /dev/null +++ b/app/data/config/mainnet-eth-keycloak/scripts/keycloak-mirror/requirements.txt @@ -0,0 +1,3 @@ +fabric +psycopg~=3.1.8 +psycopg_binary -- 2.45.2 From 08ea4b6bacc2adaaf0523062971b9b0b38817ba3 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Fri, 11 Aug 2023 23:01:28 -0500 Subject: [PATCH 10/16] Minor script fixes. --- app/data/config/mainnet-eth/scripts/run-geth.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/data/config/mainnet-eth/scripts/run-geth.sh b/app/data/config/mainnet-eth/scripts/run-geth.sh index 85730aec..7cc5d54b 100755 --- a/app/data/config/mainnet-eth/scripts/run-geth.sh +++ b/app/data/config/mainnet-eth/scripts/run-geth.sh @@ -31,7 +31,7 @@ fi $START_CMD \ $MODE_FLAGS \ - --datadir="${GETH_DATA}"\ + --datadir="${GETH_DATADIR}"\ --identity="${GETH_NODE_NAME}" \ --maxpeers=${GETH_MAX_PEERS} \ --cache=${GETH_CACHE} \ -- 2.45.2 From a4258b3aa50238f5541a1d5797a2f8ddafd05070 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Thu, 17 Aug 2023 14:19:38 -0500 Subject: [PATCH 11/16] Create new base container for pre-compiled webapps. --- .../cerc-laconic-console-host/Dockerfile | 58 +------------------ .../start-serving-app.sh | 8 --- .../cerc-webapp-base/Dockerfile | 57 ++++++++++++++++++ .../apply-webapp-config.sh | 2 +- .../container-build/cerc-webapp-base/build.sh | 9 +++ .../cerc-webapp-base/config.yml | 1 + .../cerc-webapp-base/start-serving-app.sh | 9 +++ app/data/container-image-list.txt | 1 + .../fixturenet-laconic-loaded/stack.yml | 1 + app/data/stacks/mainnet-laconic/stack.yml | 1 + 10 files changed, 82 insertions(+), 65 deletions(-) delete mode 100755 app/data/container-build/cerc-laconic-console-host/start-serving-app.sh create mode 100644 app/data/container-build/cerc-webapp-base/Dockerfile rename app/data/container-build/{cerc-laconic-console-host => cerc-webapp-base}/apply-webapp-config.sh (89%) create mode 100755 app/data/container-build/cerc-webapp-base/build.sh create mode 100644 app/data/container-build/cerc-webapp-base/config.yml create mode 100755 app/data/container-build/cerc-webapp-base/start-serving-app.sh diff --git a/app/data/container-build/cerc-laconic-console-host/Dockerfile b/app/data/container-build/cerc-laconic-console-host/Dockerfile index 7e3fc46b..95ddff94 100644 --- a/app/data/container-build/cerc-laconic-console-host/Dockerfile +++ b/app/data/container-build/cerc-laconic-console-host/Dockerfile @@ -1,69 +1,15 @@ -# Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile -# [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster -ARG VARIANT=18-bullseye -FROM node:${VARIANT} - -ARG USERNAME=node -ARG NPM_GLOBAL=/usr/local/share/npm-global +FROM cerc/webapp-base:local # This container pulls npm packages from a local registry configured via these env vars ARG CERC_NPM_REGISTRY_URL ARG CERC_NPM_AUTH_TOKEN -# Add NPM global to PATH. -ENV PATH=${NPM_GLOBAL}/bin:${PATH} -# Prevents npm from printing version warnings -ENV NPM_CONFIG_UPDATE_NOTIFIER=false - -RUN \ - # Configure global npm install location, use group to adapt to UID/GID changes - if ! cat /etc/group | grep -e "^npm:" > /dev/null 2>&1; then groupadd -r npm; fi \ - && usermod -a -G npm ${USERNAME} \ - && umask 0002 \ - && mkdir -p ${NPM_GLOBAL} \ - && touch /usr/local/etc/npmrc \ - && chown ${USERNAME}:npm ${NPM_GLOBAL} /usr/local/etc/npmrc \ - && chmod g+s ${NPM_GLOBAL} \ - && npm config -g set prefix ${NPM_GLOBAL} \ - && su ${USERNAME} -c "npm config -g set prefix ${NPM_GLOBAL}" \ - # Install eslint - && su ${USERNAME} -c "umask 0002 && npm install -g eslint" \ - && npm cache clean --force > /dev/null 2>&1 - -# [Optional] Uncomment this section to install additional OS packages. -RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ - && apt-get -y install --no-install-recommends jq - -# [Optional] Uncomment if you want to install an additional version of node using nvm -# ARG EXTRA_NODE_VERSION=10 -# RUN su node -c "source /usr/local/share/nvm/nvm.sh && nvm install ${EXTRA_NODE_VERSION}" - -# We do this to get a yq binary from the published container, for the correct architecture we're building here -COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq - -RUN mkdir -p /scripts -COPY ./apply-webapp-config.sh /scripts -COPY ./start-serving-app.sh /scripts - -# [Optional] Uncomment if you want to install more global node modules -# RUN su node -c "npm install -g " - # Configure the local npm registry RUN npm config set @cerc-io:registry ${CERC_NPM_REGISTRY_URL} \ && npm config set @lirewine:registry ${CERC_NPM_REGISTRY_URL} \ && npm config set -- ${CERC_NPM_REGISTRY_URL}:_authToken ${CERC_NPM_AUTH_TOKEN} -RUN mkdir -p /config -COPY ./config.yml /config - -# Install simple web server for now (use nginx perhaps later) -RUN yarn global add http-server - # Globally install the payload web app package RUN yarn global add @cerc-io/console-app -# Expose port for http -EXPOSE 80 - -# Default command sleeps forever so docker doesn't kill it -CMD ["/scripts/start-serving-app.sh"] +COPY ./config.yml /config diff --git a/app/data/container-build/cerc-laconic-console-host/start-serving-app.sh b/app/data/container-build/cerc-laconic-console-host/start-serving-app.sh deleted file mode 100755 index a322e5fb..00000000 --- a/app/data/container-build/cerc-laconic-console-host/start-serving-app.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -if [ -n "$CERC_SCRIPT_DEBUG" ]; then - set -x -fi -# TODO: Don't hard wire this: -webapp_files_dir=/usr/local/share/.config/yarn/global/node_modules/@cerc-io/console-app/dist/production -/scripts/apply-webapp-config.sh /config/config.yml ${webapp_files_dir} -http-server -p 80 ${webapp_files_dir} diff --git a/app/data/container-build/cerc-webapp-base/Dockerfile b/app/data/container-build/cerc-webapp-base/Dockerfile new file mode 100644 index 00000000..275a5c3c --- /dev/null +++ b/app/data/container-build/cerc-webapp-base/Dockerfile @@ -0,0 +1,57 @@ +# Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile +# [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster +ARG VARIANT=18-bullseye +FROM node:${VARIANT} + +ARG USERNAME=node +ARG NPM_GLOBAL=/usr/local/share/npm-global + +# Add NPM global to PATH. +ENV PATH=${NPM_GLOBAL}/bin:${PATH} +# Prevents npm from printing version warnings +ENV NPM_CONFIG_UPDATE_NOTIFIER=false + +RUN \ + # Configure global npm install location, use group to adapt to UID/GID changes + if ! cat /etc/group | grep -e "^npm:" > /dev/null 2>&1; then groupadd -r npm; fi \ + && usermod -a -G npm ${USERNAME} \ + && umask 0002 \ + && mkdir -p ${NPM_GLOBAL} \ + && touch /usr/local/etc/npmrc \ + && chown ${USERNAME}:npm ${NPM_GLOBAL} /usr/local/etc/npmrc \ + && chmod g+s ${NPM_GLOBAL} \ + && npm config -g set prefix ${NPM_GLOBAL} \ + && su ${USERNAME} -c "npm config -g set prefix ${NPM_GLOBAL}" \ + # Install eslint + && su ${USERNAME} -c "umask 0002 && npm install -g eslint" \ + && npm cache clean --force > /dev/null 2>&1 + +# [Optional] Uncomment this section to install additional OS packages. +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && apt-get -y install --no-install-recommends jq + +# [Optional] Uncomment if you want to install an additional version of node using nvm +# ARG EXTRA_NODE_VERSION=10 +# RUN su node -c "source /usr/local/share/nvm/nvm.sh && nvm install ${EXTRA_NODE_VERSION}" + +# We do this to get a yq binary from the published container, for the correct architecture we're building here +COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq + +RUN mkdir -p /scripts +COPY ./apply-webapp-config.sh /scripts +COPY ./start-serving-app.sh /scripts + +# [Optional] Uncomment if you want to install more global node modules +# RUN su node -c "npm install -g " + +RUN mkdir -p /config +COPY ./config.yml /config + +# Install simple web server for now (use nginx perhaps later) +RUN yarn global add http-server + +# Expose port for http +EXPOSE 80 + +# Default command sleeps forever so docker doesn't kill it +CMD ["/scripts/start-serving-app.sh"] diff --git a/app/data/container-build/cerc-laconic-console-host/apply-webapp-config.sh b/app/data/container-build/cerc-webapp-base/apply-webapp-config.sh similarity index 89% rename from app/data/container-build/cerc-laconic-console-host/apply-webapp-config.sh rename to app/data/container-build/cerc-webapp-base/apply-webapp-config.sh index bf041708..6d366805 100755 --- a/app/data/container-build/cerc-laconic-console-host/apply-webapp-config.sh +++ b/app/data/container-build/cerc-webapp-base/apply-webapp-config.sh @@ -18,7 +18,7 @@ if ! [[ -d ${webapp_files_dir} ]]; then fi # First some magic using yq to translate our yaml config file into an array of key value pairs like: # LACONIC_HOSTED_CONFIG_= -readarray -t config_kv_pair_array < <( yq '.. | select(length > 2) | ([path | join("_"), .] | join("=") )' ${config_file_name} | sed 's/^/LACONIC_HOSTED_CONFIG_/' ) +readarray -t config_kv_pair_array < <( yq '.. | ([path | join("_"), .] | join("=") )' ${config_file_name} | sort -r | sed -e '$ d' | sed 's/^/LACONIC_HOSTED_CONFIG_/' ) declare -p config_kv_pair_array # Then iterate over that kv array making the template substitution in our web app files for kv_pair_string in "${config_kv_pair_array[@]}" diff --git a/app/data/container-build/cerc-webapp-base/build.sh b/app/data/container-build/cerc-webapp-base/build.sh new file mode 100755 index 00000000..51712dc4 --- /dev/null +++ b/app/data/container-build/cerc-webapp-base/build.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Build cerc/laconic-registry-cli + +source ${CERC_CONTAINER_BASE_DIR}/build-base.sh + +# See: https://stackoverflow.com/a/246128/1701505 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +docker build -t cerc/webapp-base:local ${build_command_args} -f ${SCRIPT_DIR}/Dockerfile ${SCRIPT_DIR} diff --git a/app/data/container-build/cerc-webapp-base/config.yml b/app/data/container-build/cerc-webapp-base/config.yml new file mode 100644 index 00000000..c69b6752 --- /dev/null +++ b/app/data/container-build/cerc-webapp-base/config.yml @@ -0,0 +1 @@ +# Put config here. diff --git a/app/data/container-build/cerc-webapp-base/start-serving-app.sh b/app/data/container-build/cerc-webapp-base/start-serving-app.sh new file mode 100755 index 00000000..69fa6c22 --- /dev/null +++ b/app/data/container-build/cerc-webapp-base/start-serving-app.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/data}" + +/scripts/apply-webapp-config.sh /config/config.yml ${CERC_WEBAPP_FILES_DIR} +http-server -p 80 ${CERC_WEBAPP_FILES_DIR} diff --git a/app/data/container-image-list.txt b/app/data/container-image-list.txt index b674baf8..9ed473ba 100644 --- a/app/data/container-image-list.txt +++ b/app/data/container-image-list.txt @@ -49,3 +49,4 @@ cerc/sushiswap-v3-periphery cerc/watcher-sushiswap cerc/graph-node cerc/sushiswap-subgraphs +cerc/webapp-base diff --git a/app/data/stacks/fixturenet-laconic-loaded/stack.yml b/app/data/stacks/fixturenet-laconic-loaded/stack.yml index 3102978a..523a7091 100644 --- a/app/data/stacks/fixturenet-laconic-loaded/stack.yml +++ b/app/data/stacks/fixturenet-laconic-loaded/stack.yml @@ -21,6 +21,7 @@ npms: containers: - cerc/laconicd - cerc/laconic-registry-cli + - cerc/webapp-base - cerc/laconic-console-host pods: - fixturenet-laconicd diff --git a/app/data/stacks/mainnet-laconic/stack.yml b/app/data/stacks/mainnet-laconic/stack.yml index 51b0b8a2..e4e6781e 100644 --- a/app/data/stacks/mainnet-laconic/stack.yml +++ b/app/data/stacks/mainnet-laconic/stack.yml @@ -21,6 +21,7 @@ npms: containers: - cerc/laconicd - cerc/laconic-registry-cli + - cerc/webapp-base - cerc/laconic-console-host pods: - mainnet-laconicd -- 2.45.2 From fdd39cc41f43d836022857a5b2e3651d0f615849 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Fri, 18 Aug 2023 18:12:52 -0500 Subject: [PATCH 12/16] Add keycloak-reg-ui and keycloak-reg-api to the mainnet-eth stack. --- .../docker-compose-mainnet-eth-keycloak.yml | 16 + .../import/cerc-realm.json | 378 ++++++++++++++++-- .../config/mainnet-eth-keycloak/keycloak.env | 12 + .../config/mainnet-eth-keycloak/ui/config.yml | 4 + .../cerc-keycloak-reg-api/build.sh | 9 + .../cerc-keycloak-reg-ui/build.sh | 9 + app/data/stacks/mainnet-eth/stack.yml | 5 + 7 files changed, 396 insertions(+), 37 deletions(-) create mode 100644 app/data/config/mainnet-eth-keycloak/ui/config.yml create mode 100755 app/data/container-build/cerc-keycloak-reg-api/build.sh create mode 100755 app/data/container-build/cerc-keycloak-reg-ui/build.sh diff --git a/app/data/compose/docker-compose-mainnet-eth-keycloak.yml b/app/data/compose/docker-compose-mainnet-eth-keycloak.yml index 89447bf8..dfa9a804 100644 --- a/app/data/compose/docker-compose-mainnet-eth-keycloak.yml +++ b/app/data/compose/docker-compose-mainnet-eth-keycloak.yml @@ -31,5 +31,21 @@ services: keycloak-db: condition: service_healthy + keycloak-reg-ui: + image: cerc/keycloak-reg-ui:local + env_file: + - ../config/mainnet-eth-keycloak/keycloak.env + volumes: + - ../config/mainnet-eth-keycloak/ui:/config + ports: + - 80 + + keycloak-reg-api: + image: cerc/keycloak-reg-api:local + env_file: + - ../config/mainnet-eth-keycloak/keycloak.env + ports: + - 9292 + volumes: mainnet_eth_keycloak_db: diff --git a/app/data/config/mainnet-eth-keycloak/import/cerc-realm.json b/app/data/config/mainnet-eth-keycloak/import/cerc-realm.json index e1e9dc97..b6b6b606 100644 --- a/app/data/config/mainnet-eth-keycloak/import/cerc-realm.json +++ b/app/data/config/mainnet-eth-keycloak/import/cerc-realm.json @@ -43,6 +43,309 @@ "quickLoginCheckMilliSeconds": 1000, "maxDeltaTimeSeconds": 43200, "failureFactor": 30, + "roles": { + "realm": [ + { + "id": "0d341d8a-1f5a-4aa2-8152-1e2a9d3775bd", + "name": "uma_authorization", + "description": "${role_uma_authorization}", + "composite": false, + "clientRole": false, + "containerId": "cerc", + "attributes": {} + }, + { + "id": "7da1172a-c7d2-463d-8fb7-466a04803cc8", + "name": "offline_access", + "description": "${role_offline-access}", + "composite": false, + "clientRole": false, + "containerId": "cerc", + "attributes": {} + }, + { + "id": "211646ea-04a3-467e-9f25-f7539a405d03", + "name": "default-roles-cerc", + "description": "${role_default-roles}", + "composite": true, + "composites": { + "realm": [ + "offline_access", + "uma_authorization" + ] + }, + "clientRole": false, + "containerId": "cerc", + "attributes": {} + } + ], + "client": { + "realm-management": [ + { + "id": "caa5575f-aa68-4cd4-bf23-d4718aaf7a74", + "name": "view-identity-providers", + "description": "${role_view-identity-providers}", + "composite": false, + "clientRole": true, + "containerId": "1a91181f-823b-4cbf-9d7a-f5f097a00d73", + "attributes": {} + }, + { + "id": "b0f59506-14be-4802-85bb-91e48e10795d", + "name": "create-client", + "description": "${role_create-client}", + "composite": false, + "clientRole": true, + "containerId": "1a91181f-823b-4cbf-9d7a-f5f097a00d73", + "attributes": {} + }, + { + "id": "d1ffefd0-e63c-4473-9334-1da2023a2379", + "name": "query-users", + "description": "${role_query-users}", + "composite": false, + "clientRole": true, + "containerId": "1a91181f-823b-4cbf-9d7a-f5f097a00d73", + "attributes": {} + }, + { + "id": "9b251fe8-a743-4be4-943c-5c8fc8efb59c", + "name": "impersonation", + "description": "${role_impersonation}", + "composite": false, + "clientRole": true, + "containerId": "1a91181f-823b-4cbf-9d7a-f5f097a00d73", + "attributes": {} + }, + { + "id": "305f3c42-4385-49fa-90b0-bd35f3a6593f", + "name": "manage-users", + "description": "${role_manage-users}", + "composite": false, + "clientRole": true, + "containerId": "1a91181f-823b-4cbf-9d7a-f5f097a00d73", + "attributes": {} + }, + { + "id": "c0745551-9565-4748-92b6-7540f8e4a4c8", + "name": "view-authorization", + "description": "${role_view-authorization}", + "composite": false, + "clientRole": true, + "containerId": "1a91181f-823b-4cbf-9d7a-f5f097a00d73", + "attributes": {} + }, + { + "id": "d333ddcd-6377-48e6-bcad-83248ce42820", + "name": "manage-authorization", + "description": "${role_manage-authorization}", + "composite": false, + "clientRole": true, + "containerId": "1a91181f-823b-4cbf-9d7a-f5f097a00d73", + "attributes": {} + }, + { + "id": "3bbac383-be19-4d98-9fb0-b8ba17f73765", + "name": "view-realm", + "description": "${role_view-realm}", + "composite": false, + "clientRole": true, + "containerId": "1a91181f-823b-4cbf-9d7a-f5f097a00d73", + "attributes": {} + }, + { + "id": "547e5883-a235-49e0-bbc1-b4b089e3d4c5", + "name": "realm-admin", + "description": "${role_realm-admin}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "view-identity-providers", + "create-client", + "query-users", + "manage-users", + "impersonation", + "view-authorization", + "manage-authorization", + "view-realm", + "manage-events", + "query-realms", + "query-clients", + "manage-clients", + "view-events", + "view-clients", + "view-users", + "manage-realm", + "manage-identity-providers", + "query-groups" + ] + } + }, + "clientRole": true, + "containerId": "1a91181f-823b-4cbf-9d7a-f5f097a00d73", + "attributes": {} + }, + { + "id": "17e70842-fbc1-4c51-b79d-6ebac50c60e7", + "name": "manage-events", + "description": "${role_manage-events}", + "composite": false, + "clientRole": true, + "containerId": "1a91181f-823b-4cbf-9d7a-f5f097a00d73", + "attributes": {} + }, + { + "id": "bcf1a6f8-600b-4f27-a51a-8152f80da8a9", + "name": "query-realms", + "description": "${role_query-realms}", + "composite": false, + "clientRole": true, + "containerId": "1a91181f-823b-4cbf-9d7a-f5f097a00d73", + "attributes": {} + }, + { + "id": "bd85653b-3664-4d38-ac53-a662464bd9be", + "name": "query-clients", + "description": "${role_query-clients}", + "composite": false, + "clientRole": true, + "containerId": "1a91181f-823b-4cbf-9d7a-f5f097a00d73", + "attributes": {} + }, + { + "id": "5dbed8a7-1936-4df4-86e5-880c368b172f", + "name": "manage-clients", + "description": "${role_manage-clients}", + "composite": false, + "clientRole": true, + "containerId": "1a91181f-823b-4cbf-9d7a-f5f097a00d73", + "attributes": {} + }, + { + "id": "c4a6bd05-d72b-4206-a831-318530aa8d84", + "name": "view-events", + "description": "${role_view-events}", + "composite": false, + "clientRole": true, + "containerId": "1a91181f-823b-4cbf-9d7a-f5f097a00d73", + "attributes": {} + }, + { + "id": "19b36fad-1537-4b8e-9b1a-c5f3ef2830bf", + "name": "view-clients", + "description": "${role_view-clients}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "query-clients" + ] + } + }, + "clientRole": true, + "containerId": "1a91181f-823b-4cbf-9d7a-f5f097a00d73", + "attributes": {} + }, + { + "id": "38e7b8be-e2de-4b88-a2b3-54fa3a6bb26e", + "name": "manage-realm", + "description": "${role_manage-realm}", + "composite": false, + "clientRole": true, + "containerId": "1a91181f-823b-4cbf-9d7a-f5f097a00d73", + "attributes": {} + }, + { + "id": "11f6d5d4-d883-493b-ad1d-9818d7fd6248", + "name": "view-users", + "description": "${role_view-users}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "query-users", + "query-groups" + ] + } + }, + "clientRole": true, + "containerId": "1a91181f-823b-4cbf-9d7a-f5f097a00d73", + "attributes": {} + }, + { + "id": "93020a9b-cb4d-484a-9f65-a0a663d42fb8", + "name": "manage-identity-providers", + "description": "${role_manage-identity-providers}", + "composite": false, + "clientRole": true, + "containerId": "1a91181f-823b-4cbf-9d7a-f5f097a00d73", + "attributes": {} + }, + { + "id": "81cec017-13ec-473c-960b-1c84db230fc2", + "name": "query-groups", + "description": "${role_query-groups}", + "composite": false, + "clientRole": true, + "containerId": "1a91181f-823b-4cbf-9d7a-f5f097a00d73", + "attributes": {} + } + ], + "security-admin-console": [], + "dashboard-client": [], + "admin-cli": [], + "account-console": [], + "broker": [], + "account": [ + { + "id": "df36afa2-d09f-4235-9b80-97790f444bb3", + "name": "manage-account", + "composite": false, + "clientRole": true, + "containerId": "1ff40495-e44c-4cbc-886a-87c3ca1edc9d", + "attributes": {} + }, + { + "id": "eaaf957e-c77a-4d89-9408-ef15e31e3500", + "name": "delete-account", + "description": "${role_delete-account}", + "composite": false, + "clientRole": true, + "containerId": "1ff40495-e44c-4cbc-886a-87c3ca1edc9d", + "attributes": {} + }, + { + "id": "ba9ee05e-c4bd-44fe-b127-ba2b6a3b8cd5", + "name": "view-groups", + "composite": false, + "clientRole": true, + "containerId": "1ff40495-e44c-4cbc-886a-87c3ca1edc9d", + "attributes": {} + } + ] + } + }, + "groups": [ + { + "id": "28f8feac-7483-4c9d-9a27-81e1564e461e", + "name": "allaccess", + "path": "/allaccess", + "attributes": {}, + "realmRoles": [], + "clientRoles": {}, + "subGroups": [] + }, + { + "id": "d2a0736e-e3fc-4c23-9ebd-c56b1fd44939", + "name": "eth", + "path": "/eth", + "attributes": {}, + "realmRoles": [], + "clientRoles": {}, + "subGroups": [] + } + ], "defaultRole": { "id": "211646ea-04a3-467e-9f25-f7539a405d03", "name": "default-roles-cerc", @@ -62,8 +365,8 @@ "otpPolicyPeriod": 30, "otpPolicyCodeReusable": false, "otpSupportedApplications": [ - "totpAppGoogleName", - "totpAppFreeOTPName" + "totpAppFreeOTPName", + "totpAppGoogleName" ], "webAuthnPolicyRpEntityName": "keycloak", "webAuthnPolicySignatureAlgorithms": [ @@ -100,7 +403,8 @@ "serviceAccountClientId": "dashboard-client", "disableableCredentialTypes": [], "requiredActions": [], - "notBefore": 0 + "notBefore": 0, + "groups": [] } ], "scopeMappings": [ @@ -1186,14 +1490,14 @@ "subComponents": {}, "config": { "allowed-protocol-mapper-types": [ - "saml-user-property-mapper", - "oidc-sha256-pairwise-sub-mapper", "oidc-full-name-mapper", - "saml-role-list-mapper", - "oidc-usermodel-property-mapper", - "oidc-usermodel-attribute-mapper", + "oidc-address-mapper", + "oidc-sha256-pairwise-sub-mapper", "saml-user-attribute-mapper", - "oidc-address-mapper" + "saml-user-property-mapper", + "saml-role-list-mapper", + "oidc-usermodel-attribute-mapper", + "oidc-usermodel-property-mapper" ] } }, @@ -1237,14 +1541,14 @@ "subComponents": {}, "config": { "allowed-protocol-mapper-types": [ - "oidc-usermodel-property-mapper", "saml-role-list-mapper", "saml-user-property-mapper", "oidc-full-name-mapper", + "oidc-usermodel-attribute-mapper", "oidc-address-mapper", + "oidc-usermodel-property-mapper", "oidc-sha256-pairwise-sub-mapper", - "saml-user-attribute-mapper", - "oidc-usermodel-attribute-mapper" + "saml-user-attribute-mapper" ] } }, @@ -1312,7 +1616,7 @@ "supportedLocales": [], "authenticationFlows": [ { - "id": "43505ad9-3c8d-4f11-9f90-55bcf19e621b", + "id": "4a7f9376-0b32-482d-acf0-49080e4af5bb", "alias": "Handle Existing Account", "description": "Handle what to do if there is existing account with same email/username like authenticated identity provider", "providerId": "basic-flow", @@ -1338,7 +1642,7 @@ ] }, { - "id": "f5a8bcf1-b58f-4fd9-a0c1-4ec3933d9d64", + "id": "258bc22b-74bd-450d-b2c0-5110b16a690c", "alias": "Handle Existing Account - Alternatives - 0", "description": "Subflow of Handle Existing Account with alternative executions", "providerId": "basic-flow", @@ -1364,7 +1668,7 @@ ] }, { - "id": "b3f19451-b375-4341-8c23-f9a3b531ceb0", + "id": "b5742967-0bfc-41d8-8738-8c24934d2c7b", "alias": "Verify Existing Account by Re-authentication", "description": "Reauthentication of existing account", "providerId": "basic-flow", @@ -1390,7 +1694,7 @@ ] }, { - "id": "0db81a1c-dd36-4721-89e4-19dc7e204b56", + "id": "cc49251b-8a75-4324-abbe-50bb00384e39", "alias": "Verify Existing Account by Re-authentication - auth-otp-form - Conditional", "description": "Flow to determine if the auth-otp-form authenticator should be used or not.", "providerId": "basic-flow", @@ -1416,7 +1720,7 @@ ] }, { - "id": "e0937686-c0c4-41b2-8abd-98b5219e1953", + "id": "490a9641-0bea-425f-a04c-457d731489c0", "alias": "browser", "description": "browser based authentication", "providerId": "basic-flow", @@ -1458,7 +1762,7 @@ ] }, { - "id": "3508fa7b-a459-44ad-b56a-af9737ed86a5", + "id": "7efce4d5-b881-4e51-b390-5a40c342b185", "alias": "browser plus basic", "description": "browser based authentication", "providerId": "basic-flow", @@ -1508,7 +1812,7 @@ ] }, { - "id": "79ee49ad-20f2-4967-a9bf-ddca82c1516c", + "id": "4f16e1b0-8531-47eb-8624-4bbf877d5596", "alias": "browser plus basic forms", "description": "Username, password, otp and other auth forms.", "providerId": "basic-flow", @@ -1534,7 +1838,7 @@ ] }, { - "id": "802ce2dc-dd4a-45e6-837e-fecc17affe55", + "id": "22ddde40-03fe-425f-9dda-d7e8d108d5a3", "alias": "browser plus basic forms - auth-otp-form - Conditional", "description": "Flow to determine if the auth-otp-form authenticator should be used or not.", "providerId": "basic-flow", @@ -1560,7 +1864,7 @@ ] }, { - "id": "0f4a4d19-db06-409b-baa8-a3c8a6f52a22", + "id": "8253fd42-58bd-4536-8671-5c68522fa342", "alias": "clients", "description": "Base authentication for clients", "providerId": "client-flow", @@ -1602,7 +1906,7 @@ ] }, { - "id": "b177d3f1-dad8-4b40-ac1d-04038f0e5a7d", + "id": "04bf48cf-9568-48f4-8f17-a03af2c61419", "alias": "direct grant", "description": "OpenID Connect Resource Owner Grant", "providerId": "basic-flow", @@ -1636,7 +1940,7 @@ ] }, { - "id": "788ccbc9-c3c8-468d-8d4c-d2eb04b438a5", + "id": "61ad005d-bf91-4794-9842-3ae727a4751c", "alias": "direct grant - direct-grant-validate-otp - Conditional", "description": "Flow to determine if the direct-grant-validate-otp authenticator should be used or not.", "providerId": "basic-flow", @@ -1662,7 +1966,7 @@ ] }, { - "id": "8edd3a8f-7d9d-4029-8fd2-21a8ead2b090", + "id": "c65324a7-d836-4509-bf0c-12bd7ffcbf2b", "alias": "docker auth", "description": "Used by Docker clients to authenticate against the IDP", "providerId": "basic-flow", @@ -1680,7 +1984,7 @@ ] }, { - "id": "a67bc8ee-b99a-409f-adf5-a7d4c7f27512", + "id": "91bf5412-35f7-40ff-9374-e135aa788687", "alias": "first broker login", "description": "Actions taken after first broker login with identity provider account, which is not yet linked to any Keycloak account", "providerId": "basic-flow", @@ -1707,7 +2011,7 @@ ] }, { - "id": "ffe8dad9-6998-4358-ab2c-061cf7235d53", + "id": "4112f733-14e0-404b-978e-335ecda4a88e", "alias": "first broker login - Alternatives - 0", "description": "Subflow of first broker login with alternative executions", "providerId": "basic-flow", @@ -1734,7 +2038,7 @@ ] }, { - "id": "26133bdd-6657-449d-a823-73519956b272", + "id": "fc661cc2-942d-4596-84e7-0ab62c6dada2", "alias": "forms", "description": "Username, password, otp and other auth forms.", "providerId": "basic-flow", @@ -1760,7 +2064,7 @@ ] }, { - "id": "57620e5a-f7cd-4e88-ac51-d78e91ff7868", + "id": "06555841-cc79-4f16-8497-6c107896e07a", "alias": "forms - auth-otp-form - Conditional", "description": "Flow to determine if the auth-otp-form authenticator should be used or not.", "providerId": "basic-flow", @@ -1786,7 +2090,7 @@ ] }, { - "id": "cffbb5df-de0a-49ed-9136-296a877ab175", + "id": "850ed202-6ac8-4dbc-80dd-ef181327bc23", "alias": "http challenge", "description": "An authentication flow based on challenge-response HTTP Authentication Schemes", "providerId": "basic-flow", @@ -1828,7 +2132,7 @@ ] }, { - "id": "6ac5a9df-dacb-462c-9b12-207470e9fcbf", + "id": "b5a45a81-fdc4-4473-9194-595b5b09f817", "alias": "registration", "description": "registration flow", "providerId": "basic-flow", @@ -1847,7 +2151,7 @@ ] }, { - "id": "27e40f78-ce1e-4ad4-9b48-88a8bf9c8d92", + "id": "0f2d666e-7413-495e-aeb5-abed50c497f4", "alias": "registration form", "description": "registration form", "providerId": "form-flow", @@ -1889,7 +2193,7 @@ ] }, { - "id": "31340e3b-f6c7-49ce-94ac-f28213b84be6", + "id": "4cbd3b65-cec8-4a0a-8566-50336ad16dc8", "alias": "reset credentials", "description": "Reset credentials for a user if they forgot their password or something", "providerId": "basic-flow", @@ -1931,7 +2235,7 @@ ] }, { - "id": "aee4a6d9-caab-463e-ad62-48aba91a4098", + "id": "c027601d-55dd-4c36-9821-816815689e48", "alias": "reset credentials - reset-otp - Conditional", "description": "Flow to determine if the reset-otp authenticator should be used or not.", "providerId": "basic-flow", @@ -1957,7 +2261,7 @@ ] }, { - "id": "4052bdf6-9b94-42a1-b199-0c14ffe67ac5", + "id": "76a19a9d-bbe9-4274-b743-ee5a001e7cff", "alias": "saml ecp", "description": "SAML ECP Profile Authentication Flow", "providerId": "basic-flow", @@ -1977,14 +2281,14 @@ ], "authenticatorConfig": [ { - "id": "4bc95f52-8c28-449c-830b-a4ffc3340399", + "id": "6428d38a-d80b-4cc0-89a2-698c7eb40fbb", "alias": "create unique user config", "config": { "require.password.update.after.registration": "false" } }, { - "id": "367a56fc-c128-43f8-85d5-50ceae63b7aa", + "id": "d0dbc8d3-d2e5-4de3-bdb6-83c6a0b2f904", "alias": "review profile config", "config": { "update.profile.on.first.login": "missing" @@ -2076,7 +2380,7 @@ "cibaInterval": "5", "realmReusableOtpCode": "false" }, - "keycloakVersion": "20.0.2", + "keycloakVersion": "20.0.5", "userManagedAccessAllowed": false, "clientProfiles": { "profiles": [] diff --git a/app/data/config/mainnet-eth-keycloak/keycloak.env b/app/data/config/mainnet-eth-keycloak/keycloak.env index 0bc7bf15..5b4280ab 100644 --- a/app/data/config/mainnet-eth-keycloak/keycloak.env +++ b/app/data/config/mainnet-eth-keycloak/keycloak.env @@ -15,3 +15,15 @@ KEYCLOAK_ADMIN=admin KEYCLOAK_ADMIN_PASSWORD=admin X_API_CHECK_REALM=cerc X_API_CHECK_CLIENT_ID="%api_key%" + + +# keycloak-reg-api +CERC_KCUSERREG_LISTEN_PORT=9292 +CERC_KCUSERREG_LISTEN_ADDR='0.0.0.0' +CERC_KCUSERREG_API_URL='http://keycloak:8080/auth' +CERC_KCUSERREG_REG_USER="${KEYCLOAK_ADMIN}" +CERC_KCUSERREG_REG_PW="${KEYCLOAK_ADMIN_PASSWORD}" +CERC_KCUSERREG_REG_CLIENT_ID='admin-cli' +CERC_KCUSERREG_TARGET_REALM=cerc +CERC_KCUSERREG_TARGET_GROUPS=eth +CERC_KCUSERREG_CREATE_ENABLED=true diff --git a/app/data/config/mainnet-eth-keycloak/ui/config.yml b/app/data/config/mainnet-eth-keycloak/ui/config.yml new file mode 100644 index 00000000..6f38a61b --- /dev/null +++ b/app/data/config/mainnet-eth-keycloak/ui/config.yml @@ -0,0 +1,4 @@ +web: + path: '' +api: + url: 'http://keycloak-reg-api:9292' diff --git a/app/data/container-build/cerc-keycloak-reg-api/build.sh b/app/data/container-build/cerc-keycloak-reg-api/build.sh new file mode 100755 index 00000000..c591c2f0 --- /dev/null +++ b/app/data/container-build/cerc-keycloak-reg-api/build.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Build cerc/keycloak-reg-api + +source ${CERC_CONTAINER_BASE_DIR}/build-base.sh + +# See: https://stackoverflow.com/a/246128/1701505 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +docker build -t cerc/keycloak-reg-api:local ${build_command_args} ${CERC_REPO_BASE_DIR}/keycloak-reg-api diff --git a/app/data/container-build/cerc-keycloak-reg-ui/build.sh b/app/data/container-build/cerc-keycloak-reg-ui/build.sh new file mode 100755 index 00000000..3124dae6 --- /dev/null +++ b/app/data/container-build/cerc-keycloak-reg-ui/build.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Build cerc/keycloak-reg-ui + +source ${CERC_CONTAINER_BASE_DIR}/build-base.sh + +# See: https://stackoverflow.com/a/246128/1701505 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +docker build -t cerc/keycloak-reg-ui:local ${build_command_args} ${CERC_REPO_BASE_DIR}/keycloak-reg-ui diff --git a/app/data/stacks/mainnet-eth/stack.yml b/app/data/stacks/mainnet-eth/stack.yml index 11885cef..be63e972 100644 --- a/app/data/stacks/mainnet-eth/stack.yml +++ b/app/data/stacks/mainnet-eth/stack.yml @@ -5,12 +5,17 @@ repos: - github.com/cerc-io/go-ethereum - github.com/cerc-io/lighthouse - github.com/dboreham/foundry + - git.vdb.to/cerc-io/keycloak-reg-api + - git.vdb.to/cerc-io/keycloak-reg-ui containers: - cerc/go-ethereum - cerc/lighthouse - cerc/lighthouse-cli - cerc/foundry - cerc/keycloak + - cerc/webapp-base + - cerc/keycloak-reg-api + - cerc/keycloak-reg-ui pods: - mainnet-eth - mainnet-eth-keycloak -- 2.45.2 From da4f164a1be44cf785bde68319c5a82fde718ea6 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Sat, 19 Aug 2023 04:04:16 +0000 Subject: [PATCH 13/16] Fix port --- app/data/compose/docker-compose-mainnet-eth.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/data/compose/docker-compose-mainnet-eth.yml b/app/data/compose/docker-compose-mainnet-eth.yml index 04f5b236..1a6fc529 100644 --- a/app/data/compose/docker-compose-mainnet-eth.yml +++ b/app/data/compose/docker-compose-mainnet-eth.yml @@ -40,7 +40,7 @@ services: restart: always hostname: mainnet-eth-lighthouse-1 healthcheck: - test: ["CMD", "wget", "--tries=1", "--connect-timeout=1", "--quiet", "-O", "-", "http://localhost:8001/eth/v2/beacon/blocks/head"] + test: ["CMD", "wget", "--tries=1", "--connect-timeout=1", "--quiet", "-O", "-", "http://localhost:5052/eth/v2/beacon/blocks/head"] interval: 30s timeout: 10s retries: 10 -- 2.45.2 From 9895434856ad7d39ebe6640d772a36376dc7a776 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Tue, 22 Aug 2023 14:01:39 -0500 Subject: [PATCH 14/16] Get Grafana password from env. --- app/data/compose/docker-compose-mainnet-eth-metrics.yml | 6 ++++-- app/data/config/mainnet-eth-metrics/metrics.env | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 app/data/config/mainnet-eth-metrics/metrics.env diff --git a/app/data/compose/docker-compose-mainnet-eth-metrics.yml b/app/data/compose/docker-compose-mainnet-eth-metrics.yml index c3978018..03c223e7 100644 --- a/app/data/compose/docker-compose-mainnet-eth-metrics.yml +++ b/app/data/compose/docker-compose-mainnet-eth-metrics.yml @@ -6,6 +6,8 @@ services: depends_on: mainnet-eth-geth-1: condition: service_healthy + env_file: + - ../config/mainnet-eth-metrics/metrics.env volumes: - ../config/mainnet-eth-metrics/prometheus/etc:/etc/prometheus ports: @@ -13,8 +15,8 @@ services: grafana: restart: always image: grafana/grafana - environment: - - GF_SECURITY_ADMIN_PASSWORD=changeme6325 + env_file: + - ../config/mainnet-eth-metrics/metrics.env volumes: - ../config/mainnet-eth-metrics/grafana/etc/provisioning/dashboards:/etc/grafana/provisioning/dashboards - ../config/mainnet-eth-metrics/grafana/etc/provisioning/datasources:/etc/grafana/provisioning/datasources diff --git a/app/data/config/mainnet-eth-metrics/metrics.env b/app/data/config/mainnet-eth-metrics/metrics.env new file mode 100644 index 00000000..1cb51f6b --- /dev/null +++ b/app/data/config/mainnet-eth-metrics/metrics.env @@ -0,0 +1,2 @@ +# grafana +GF_SECURITY_ADMIN_PASSWORD=changeme6325 -- 2.45.2 From 5f570a8a397dc22bff3c4a38306c8efbd8f3e113 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 23 Aug 2023 14:14:56 -0500 Subject: [PATCH 15/16] Add example nginx config. --- .../config/mainnet-eth-keycloak/nginx.example | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 app/data/config/mainnet-eth-keycloak/nginx.example diff --git a/app/data/config/mainnet-eth-keycloak/nginx.example b/app/data/config/mainnet-eth-keycloak/nginx.example new file mode 100644 index 00000000..67095551 --- /dev/null +++ b/app/data/config/mainnet-eth-keycloak/nginx.example @@ -0,0 +1,107 @@ +server { + listen 80; + server_name my.example.com; + + # See: https://github.com/acmesh-official/acme.sh/wiki/Stateless-Mode + # and https://datatracker.ietf.org/doc/html/rfc8555 + location ~ ^/\.well-known/acme-challenge/([-_a-zA-Z0-9]+)$ { + default_type text/plain; + return 200 "$1.MY_ACCOUNT_THUMBPRINT_GOES_HERE"; + } + + location / { + return 301 https://$host$request_uri; + } +} + +upstream geth-pool { + keepalive 100; + hash $user_id consistent; + server server-a:8545; + server server-b:8545; + server server-c:8545; +} + +# self-reg happens on one server for clarity +upstream reg-ui-pool { + keepalive 100; + server server-a:8085; +} + +upstream reg-api-pool { + keepalive 100; + server server-a:8086; +} + +# auth uses server-a if available +upstream auth-pool { + keepalive 100; + server server-a:8080; + server server-b:8080 backup; + server server-c:8080 backup; +} + +log_format upstreamlog '[$time_local] $remote_addr $user_id - $server_name $host to: $upstream_addr: $request $status upstream_response_time $upstream_response_time msec $msec request_time $request_time'; +proxy_cache_path /var/cache/nginx/auth_cache levels=1 keys_zone=auth_cache:1m max_size=5m inactive=60m; +server { + listen 443 ssl http2; + server_name my.example.com; + access_log /var/log/nginx/my.example.com-access.log upstreamlog; + error_log /var/log/nginx/my.example.com-error.log; + + ssl_certificate /etc/nginx/ssl/my.example.com/cert.pem; + ssl_certificate_key /etc/nginx/ssl/my.example.com/key.pem; + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + + #rewrite ^/?$ /newuser/; + rewrite ^/?$ https://www.example.com/; + + + # geth-pool ETH API + location ~ ^/v1/eth/?([^/]*)$ { + set $apiKey $1; + if ($apiKey = '') { + set $apiKey $http_X_API_KEY; + } + auth_request /auth; + auth_request_set $user_id $sent_http_x_user_id; + rewrite /.*$ / break; + + client_max_body_size 3m; + client_body_buffer_size 3m; + proxy_buffer_size 32k; + proxy_buffers 16 32k; + proxy_busy_buffers_size 96k; + + proxy_pass http://geth-pool; + proxy_set_header X-Original-Remote-Addr $remote_addr; + proxy_set_header X-User-Id $user_id; + } + + # keycloak + location = /auth { + internal; + proxy_cache auth_cache; + proxy_cache_key "$apiKey"; + proxy_cache_valid 200 300s; + proxy_cache_valid 401 30s; + proxy_pass http://auth-pool/auth/realms/cerc/check?memberOf=eth&apiKey=$apiKey; + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-URI $request_uri; + proxy_set_header X-Original-Remote-Addr $remote_addr; + proxy_set_header X-Original-Host $host; + } + + location /newuser/ { + proxy_pass http://reg-ui-pool/; + } + + location /user-api/ { + proxy_pass http://reg-api-pool/; + } +} -- 2.45.2 From 90576e80d36c993019d096a073aa21def410e2ed Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 23 Aug 2023 17:54:22 -0500 Subject: [PATCH 16/16] We want user_id for this sort of thing. --- app/data/config/mainnet-eth-keycloak/keycloak.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/data/config/mainnet-eth-keycloak/keycloak.env b/app/data/config/mainnet-eth-keycloak/keycloak.env index 5b4280ab..f37fdd30 100644 --- a/app/data/config/mainnet-eth-keycloak/keycloak.env +++ b/app/data/config/mainnet-eth-keycloak/keycloak.env @@ -14,7 +14,7 @@ KC_HOSTNAME_STRICT_HTTPS="false" KEYCLOAK_ADMIN=admin KEYCLOAK_ADMIN_PASSWORD=admin X_API_CHECK_REALM=cerc -X_API_CHECK_CLIENT_ID="%api_key%" +X_API_CHECK_CLIENT_ID="%user_id%" # keycloak-reg-api -- 2.45.2