Support uploading config files. (#14)

This adds a new `/upload/config` endpoint to the API for uploading encrypted configuration files for later use by the deployer.

The payload takes the form:

```
authorized:
  - accounta
  - accountb
config:
  env:
    FOO: bar
    BAR: baz
```

The request is the encrypted using the deployer's public key (discoverable from its `WebappDeployer` record).  This is handled automatically by `laconic-so` but can also be handled manually using standard CLI tools like `gpg` and `curl`.

For example:

```
# Get the key
$ laconic -c ~/.laconic/testnet-a-cercio.yml registry name resolve lrn://laconic/deployers/webapp-deployer-api.dev.vaasl.io | jq -r '.[0].attributes.publicKey' | base64 -d > webapp-deployer-api.dev.vaasl.io.pgp.pub

# Import it
$ gpg --import webapp-deployer-api.dev.vaasl.io.pgp.pub

# Encrypt your config file.
$ gpg --yes --encrypt --recipient webapp-deployer-api.dev.vaasl.io --trust-model always config.yaml

# Post it
$ curl -s -X POST -d '@config.yaml.gpg' https://webapp-deployer-api.dev.vaasl.io/upload/config | jq
{
  "id": "B56C65AB96B741B7B219520A3ABFCD10"
}
```

Reviewed-on: #14
Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com>
Co-committed-by: Thomas E Lackey <telackey@bozemanpass.com>
This commit is contained in:
Thomas E Lackey 2024-08-27 19:44:52 +00:00 committed by Thomas E Lackey
parent 30b349ea49
commit 3372ce29f3
8 changed files with 232 additions and 23 deletions

View File

@ -37,14 +37,19 @@
"license": "Apache-2.0",
"dependencies": {
"@cerc-io/laconic-sdk": "^0.1.15",
"@openpgp/web-stream-tools": "^0.1.3",
"body-parser": "^1.20.2",
"express": "^4.18.2",
"express-async-handler": "^1.2.0",
"express-serve-static-core": "^0.1.1",
"js-yaml": "^4.1.0",
"json-stable-stringify": "^1.1.1",
"tslib": "~2.6"
"openpgp": "^5.11.2",
"tslib": "~2.6",
"yaml": "^2.5.0"
},
"volta": {
"node": "20.10.0"
}
},
"packageManager": "yarn@1.22.22+sha1.ac34549e6aa8e7ead463a7407e1c7390f61a6610"
}

15
run.sh
View File

@ -71,6 +71,10 @@ if [[ "$CLEAN_LOGS" == "true" ]] && [[ -n "$LOG_DIR" ]]; then
rm -rf ${LOG_DIR}/*
fi
if [[ ! -d "${UPLOAD_DIRECTORY}" ]]; then
mkdir -p "${UPLOAD_DIRECTORY}"
fi
STORAGE_DRIVER="${STORAGE_DRIVER}"
if [[ -z "${STORAGE_DRIVER}" ]]; then
if [[ "true" == "`is_privileged`" ]]; then
@ -118,6 +122,8 @@ while true; do
--state-file "${DEPLOYMENTS_DIR}/autoremove.state" \
--include-tags "$INCLUDE_TAGS" \
--exclude-tags "$EXCLUDE_TAGS" \
--lrn "$LRN" \
--min-required-payment ${MIN_REQUIRED_PAYMENT:-0} \
$EXTRA_UNDEPLOY_OPTS \
$UPDATE_OPTS \
--discover
@ -125,7 +131,7 @@ while true; do
if [ $rc -eq 0 ]; then
echo "############ UNDEPLOY SUCCESS #############"
else
echo "############ UNDEPLOY FAILURE $rc #############"
echo "############ UNDEPLOY FAILURE STATUS $rc #############"
fi
echo "############ DEPLOY #############"
@ -141,6 +147,11 @@ while true; do
--include-tags "$INCLUDE_TAGS" \
--exclude-tags "$EXCLUDE_TAGS" \
--fqdn-policy "${FQDN_POLICY:-prohibit}" \
--lrn "$LRN" \
--min-required-payment ${MIN_REQUIRED_PAYMENT:-0} \
--config-upload-dir "$UPLOAD_DIRECTORY" \
--private-key-file "$OPENPGP_PRIVATE_KEY_FILE"
--private-key-passphrase "$OPENPGP_PASSPHRASE"
$LOG_OPTS \
$EXTRA_DEPLOY_OPTS \
$UPDATE_OPTS \
@ -149,7 +160,7 @@ while true; do
if [ $rc -eq 0 ]; then
echo "############ DEPLOY SUCCESS #############"
else
echo "############ DEPLOY FAILURE $rc #############"
echo "############ DEPLOY FAILURE STATUS $rc #############"
fi
# Cleanup any build leftovers

View File

@ -13,11 +13,15 @@ export const Config = {
LISTEN_PORT: parseInt(process.env.LISTEN_PORT || '9555'),
LISTEN_ADDR: process.env.LISTEN_ADDR || '0.0.0.0',
LACONIC_CONFIG: process.env.LACONIC_CONFIG || '/etc/config/laconic.yml',
UPLOAD_DIRECTORY: process.env.UPLOAD_DIRECTORY || '/srv/uploads',
DEPLOYER_STATE:
process.env.DEPLOYER_STATE || '/srv/deployments/autodeploy.state',
UNDEPLOYER_STATE:
process.env.UNDEPLOYER_STATE || '/srv/deployments/autoundeploy.state',
BUILD_LOGS: process.env.BUILD_LOGS || '/srv/logs',
UPLOAD_MAX_SIZE: process.env.BUILD_LOGS || '1MB',
OPENPGP_PASSPHRASE: process.env.OPENPGP_PASSPHRASE,
OPENPGP_PRIVATE_KEY_FILE: process.env.OPENPGP_PRIVATE_KEY_FILE,
};
export const getRegistry = (): Registry => {

View File

@ -152,15 +152,22 @@ export class RegHelper {
const status = new RequestStatus(r.id, r.createTime);
ret.push(status);
const app = await this.getRecord(r.attributes.application);
if (!app) {
status.lastState = 'ERROR';
continue;
}
status.app = r.attributes.application;
const hostname = r.attributes.dns ?? generateHostnameForApp(app);
if (deploymentsByRequest.has(r.id)) {
const deployment = deploymentsByRequest.get(r.id);
status.url = deployment.attributes.url;
status.lastUpdate = deployment.createTime;
const shortHost = new URL(status.url).host.split('.').shift();
if (!latestByHostname.has(shortHost)) {
latestByHostname.set(shortHost, status);
if (!latestByHostname.has(hostname)) {
latestByHostname.set(hostname, status);
}
status.deployment = deployment.names ? deployment.names[0] : null;
if (status.deployment) {
@ -176,19 +183,12 @@ export class RegHelper {
continue;
}
const app = await this.getRecord(r.attributes.application);
if (!app) {
status.lastState = 'ERROR';
continue;
}
const shortHost = r.attributes.dns ?? generateHostnameForApp(app);
if (latestByHostname.has(shortHost)) {
if (latestByHostname.has(hostname)) {
status.lastState = 'CANCELLED';
continue;
}
latestByHostname.set(shortHost, status);
latestByHostname.set(hostname, status);
}
return ret;

View File

@ -1,13 +1,18 @@
import express from 'express';
import bodyParser from 'body-parser';
import {existsSync, readdirSync, readFileSync} from 'fs';
import {Config} from './config.js';
import {RegHelper} from './deployments.js';
import { Uploader } from './upload.js';
const app = express();
app.use(express.json());
const configUploader = new Uploader(Config.UPLOAD_DIRECTORY);
const configUploadParser = bodyParser.raw({limit: Config.UPLOAD_MAX_SIZE, type: "*/*"})
app.use(function (_req, res, next) {
res.header('Access-Control-Allow-Origin', '*');
res.header('Access-Control-Allow-Headers', 'Origin, X-Requested-With, Content-Type, Accept',);
@ -95,6 +100,18 @@ app.get('/:id/log', async (req, res) => {
}
});
app.post('/upload/config', configUploadParser, async (req, res) => {
try {
const id = await configUploader.upload(req.body);
res.json({
id
});
} catch (e) {
console.error(e);
res.sendStatus(500);
}
});
// deprecated
app.get('/log/:id', async (req, res) => {
try {

91
src/upload.ts Normal file
View File

@ -0,0 +1,91 @@
import crypto from 'crypto';
import fs from 'fs';
import assert from 'node:assert';
import openpgp from 'openpgp';
import YAML from 'yaml';
import { atob } from 'node:buffer';
import { Config } from './config.js';
let privateKey: openpgp.PrivateKey | null = null;
const loadPrivateKey = async () => {
if (null == privateKey) {
privateKey = await openpgp.decryptKey({
privateKey: await openpgp.readPrivateKey({
binaryKey: fs.readFileSync(Config.OPENPGP_PRIVATE_KEY_FILE)
}),
passphrase: Config.OPENPGP_PASSPHRASE,
});
}
return privateKey;
}
const randomId = (): string =>
crypto
.randomUUID({ disableEntropyCache: true })
.replaceAll('-', '')
.toUpperCase();
const validateConfig = (obj): undefined => {
assert(obj.authorized, "'authorized' is required");
assert(Array.isArray(obj.authorized), "'authorized' must be an array");
assert(obj.authorized.length >= 1, "'authorized' cannot be empty");
assert(obj.config, "'config' is required");
};
export const b64ToBytes = (base64): Uint8Array => {
const binaryString = atob(base64);
const bytes = new Uint8Array(binaryString.length);
for (let i = 0; i < binaryString.length; i++) {
bytes[i] = binaryString.charCodeAt(i);
}
return bytes;
};
const decrypt = async (binaryMessage: Uint8Array): Promise<any> => {
const message = await openpgp.readMessage({
binaryMessage,
});
const { data } = await openpgp.decrypt({
message,
decryptionKeys: await loadPrivateKey(),
});
const config = data.toString();
return config.charAt(0) === '{' ? JSON.parse(config) : YAML.parse(config);
};
export class Uploader {
directory: string;
constructor(dir: string) {
this.directory = dir;
}
async upload(body: string | Uint8Array): Promise<string> {
let raw: any;
try {
raw = b64ToBytes(body);
} catch {
raw = body;
}
// We decrypt only to make sure the content is valid.
// Once we know it is good, we want to store the encrypted copy.
const obj = await decrypt(raw);
validateConfig(obj);
let id: string;
let destination: string;
do {
id = randomId();
destination = `${this.directory}/${id}`;
} while (fs.existsSync(destination));
console.log(`Wrote config to: ${destination}`);
fs.writeFileSync(destination, raw);
return id;
}
}

View File

@ -17,7 +17,8 @@
"noUnusedParameters": true,
"noImplicitAny": false,
"noImplicitThis": false,
"strictNullChecks": false
"strictNullChecks": false,
"skipLibCheck": true
},
"include": ["src/**/*", "__tests__/**/*"]
}

View File

@ -1146,6 +1146,11 @@
resolved "https://registry.yarnpkg.com/@octetstream/promisify/-/promisify-2.0.2.tgz#29ac3bd7aefba646db670227f895d812c1a19615"
integrity sha512-7XHoRB61hxsz8lBQrjC1tq/3OEIgpvGWg6DKAdwi7WRzruwkmsdwmOoUXbU4Dtd4RSOMDwed0SkP3y8UlMt1Bg==
"@openpgp/web-stream-tools@^0.1.3":
version "0.1.3"
resolved "https://registry.yarnpkg.com/@openpgp/web-stream-tools/-/web-stream-tools-0.1.3.tgz#a9750f12a634b5a15e711b6c1de559511fb53732"
integrity sha512-mT/ds43cH6c+AO5RFpxs+LkACr7KjC3/dZWHrP6KPrWJu4uJ/XJ+p7telaoYiqUfdjiiIvdNSOfhezW9fkmboQ==
"@pkgjs/parseargs@^0.11.0":
version "0.11.0"
resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33"
@ -1657,6 +1662,16 @@ array-union@^2.1.0:
resolved "https://registry.yarnpkg.com/array-union/-/array-union-2.1.0.tgz#b798420adbeb1de828d84acd8a2e23d3efe85e8d"
integrity sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==
asn1.js@^5.0.0:
version "5.4.1"
resolved "https://registry.yarnpkg.com/asn1.js/-/asn1.js-5.4.1.tgz#11a980b84ebb91781ce35b0fdc2ee294e3783f07"
integrity sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA==
dependencies:
bn.js "^4.0.0"
inherits "^2.0.1"
minimalistic-assert "^1.0.0"
safer-buffer "^2.1.0"
axios@^0.26.1:
version "0.26.1"
resolved "https://registry.yarnpkg.com/axios/-/axios-0.26.1.tgz#1ede41c51fcf51bbbd6fd43669caaa4f0495aaa9"
@ -1787,7 +1802,7 @@ blakejs@^1.1.0:
resolved "https://registry.yarnpkg.com/blakejs/-/blakejs-1.2.1.tgz#5057e4206eadb4a97f7c0b6e197a505042fc3814"
integrity sha512-QXUSXI3QVc/gJME0dBpXrag1kbzOqCjCX8/b54ntNyW6sjtoqxqRk3LTmXzaJoh71zMsDCjM+47jS7XiwN/+fQ==
bn.js@^4.11.0, bn.js@^4.11.8, bn.js@^4.11.9:
bn.js@^4.0.0, bn.js@^4.11.0, bn.js@^4.11.8, bn.js@^4.11.9:
version "4.12.0"
resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-4.12.0.tgz#775b3f278efbb9718eec7361f483fb36fbbfea88"
integrity sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==
@ -1815,6 +1830,24 @@ body-parser@1.20.1:
type-is "~1.6.18"
unpipe "1.0.0"
body-parser@^1.20.2:
version "1.20.2"
resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.2.tgz#6feb0e21c4724d06de7ff38da36dad4f57a747fd"
integrity sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==
dependencies:
bytes "3.1.2"
content-type "~1.0.5"
debug "2.6.9"
depd "2.0.0"
destroy "1.2.0"
http-errors "2.0.0"
iconv-lite "0.4.24"
on-finished "2.4.1"
qs "6.11.0"
raw-body "2.5.2"
type-is "~1.6.18"
unpipe "1.0.0"
brace-expansion@^1.1.7:
version "1.1.11"
resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd"
@ -2051,7 +2084,7 @@ content-disposition@0.5.4:
dependencies:
safe-buffer "5.2.1"
content-type@~1.0.4:
content-type@~1.0.4, content-type@~1.0.5:
version "1.0.5"
resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.5.tgz#8b773162656d1d1086784c8f23a54ce6d73d7918"
integrity sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==
@ -3916,6 +3949,13 @@ onetime@^5.1.2:
dependencies:
mimic-fn "^2.1.0"
openpgp@^5.11.2:
version "5.11.2"
resolved "https://registry.yarnpkg.com/openpgp/-/openpgp-5.11.2.tgz#2c035a26b13feb3b0bb5180718ec91c8e65cc686"
integrity sha512-f8dJFVLwdkvPvW3VPFs6q9Vs2+HNhdvwls7a/MIFcQUB+XiQzRe7alfa3RtwfGJU7oUDDMAWPZ0nYsHa23Az+A==
dependencies:
asn1.js "^5.0.0"
optionator@^0.9.3:
version "0.9.3"
resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.3.tgz#007397d44ed1872fdc6ed31360190f81814e2c64"
@ -4159,6 +4199,16 @@ raw-body@2.5.1:
iconv-lite "0.4.24"
unpipe "1.0.0"
raw-body@2.5.2:
version "2.5.2"
resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.5.2.tgz#99febd83b90e08975087e8f1f9419a149366b68a"
integrity sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==
dependencies:
bytes "3.1.2"
http-errors "2.0.0"
iconv-lite "0.4.24"
unpipe "1.0.0"
react-is@^18.0.0:
version "18.2.0"
resolved "https://registry.yarnpkg.com/react-is/-/react-is-18.2.0.tgz#199431eeaaa2e09f86427efbb4f1473edb47609b"
@ -4272,7 +4322,7 @@ safe-buffer@5.2.1, safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, s
resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"
integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==
"safer-buffer@>= 2.1.2 < 3":
"safer-buffer@>= 2.1.2 < 3", safer-buffer@^2.1.0:
version "2.1.2"
resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a"
integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==
@ -4464,7 +4514,16 @@ string-length@^4.0.1:
char-regex "^1.0.2"
strip-ansi "^6.0.0"
"string-width-cjs@npm:string-width@^4.2.0", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3:
"string-width-cjs@npm:string-width@^4.2.0":
version "4.2.3"
resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
dependencies:
emoji-regex "^8.0.0"
is-fullwidth-code-point "^3.0.0"
strip-ansi "^6.0.1"
string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3:
version "4.2.3"
resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
@ -4489,7 +4548,14 @@ string_decoder@^1.1.1:
dependencies:
safe-buffer "~5.2.0"
"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1:
"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
version "6.0.1"
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
dependencies:
ansi-regex "^5.0.1"
strip-ansi@^6.0.0, strip-ansi@^6.0.1:
version "6.0.1"
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
@ -4754,7 +4820,16 @@ wif@^2.0.6:
dependencies:
bs58check "<3.0.0"
"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0:
"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
version "7.0.0"
resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
dependencies:
ansi-styles "^4.0.0"
string-width "^4.1.0"
strip-ansi "^6.0.0"
wrap-ansi@^7.0.0:
version "7.0.0"
resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
@ -4805,6 +4880,11 @@ yallist@^4.0.0:
resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72"
integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==
yaml@^2.5.0:
version "2.5.0"
resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.5.0.tgz#c6165a721cf8000e91c36490a41d7be25176cf5d"
integrity sha512-2wWLbGbYDiSqqIKoPjar3MPgB94ErzCtrNE1FdqGuaO0pi2JGjmE8aW8TDZwzU7vuxcGRdL/4gPQwQ7hD5AMSw==
yargs-parser@^21.0.1, yargs-parser@^21.1.1:
version "21.1.1"
resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-21.1.1.tgz#9096bceebf990d21bb31fa9516e0ede294a77d35"