From 88604959f48ce6879ad79ecc66268e73113932c8 Mon Sep 17 00:00:00 2001 From: Edd Date: Mon, 11 Apr 2022 12:12:07 +0100 Subject: [PATCH 01/10] Remove unused publish step --- .github/workflows/publish.yml | 39 ----------------------------------- 1 file changed, 39 deletions(-) delete mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index 76c9e4d1f..000000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -name: Publish - -'on': - push: - tags: - - 'v[0-9]+.[0-9]+.[0-9]+' - - 'v[0-9]+.[0-9]+.[0-9]+-pre[0-9]+' - -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - node-version: [14.x] - steps: - - name: Check out code - uses: actions/checkout@v2 - - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v2 - with: - node-version: ${{ matrix.node-version }} - registry-url: https://registry.npmjs.org/ - - - name: Install - run: | - yarn install --frozen-lockfile - - name: Build - run: | - yarn build - - name: Test - run: | - yarn test:coverage - - name: Push release to npm (public) - env: - NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}} - run: | - yarn publish --access public . From 463bb2e360a037ccdad9a18dbf238f82674fda93 Mon Sep 17 00:00:00 2001 From: Edd Date: Mon, 11 Apr 2022 12:12:40 +0100 Subject: [PATCH 02/10] Add fleek file for stats, and script to run it if affected --- apps/stats/.fleek.json | 15 +++++++++ tools/fleek-deploy.js | 76 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 apps/stats/.fleek.json create mode 100755 tools/fleek-deploy.js diff --git a/apps/stats/.fleek.json b/apps/stats/.fleek.json new file mode 100644 index 000000000..cbfd910f9 --- /dev/null +++ b/apps/stats/.fleek.json @@ -0,0 +1,15 @@ +{ + "site": { + "id": "2365cbf8-20f8-4d3a-8014-19e813eab683", + "team": "886f256a-fd19-40d5-9551-bcd71a7c8100", + "platform": "ipfs", + "source": "github", + "name": "vega-status" + }, + "build": { + "baseDir": "", + "publicDir": "dist/apps/stats/", + "command": "yarn && yarn nx run stats:build", + "rootDir": "" + } +} diff --git a/tools/fleek-deploy.js b/tools/fleek-deploy.js new file mode 100755 index 000000000..3817f23c4 --- /dev/null +++ b/tools/fleek-deploy.js @@ -0,0 +1,76 @@ +#!/usr/bin/env node + +/** + * Runs the fleek deploy process based on nx:affected, but only for sites + * that have a .fleek.json file + * + * @author Edd + */ + +// Fleek CLI requires this variable to be set +if (!process.env['FLEEK_API_KEY']) { + console.error('Error: FLEEK_API_KEY must be set'); + process.exit(1); +} + +const { existsSync } = require('fs'); +const { execSync } = require('child_process'); + +// The folder containing NX projects +const projectPath = './apps/'; +// The Fleek project file, the existence indicates a deployed app +const fleekFile = '.fleek.json'; +// Command to run in each app that needs to be deployed +const deployCommand = 'npx @fleekhq/fleek-cli@0.1.8 site:deploy'; + +// Await piped input +process.stdin.resume(); +process.stdin.setEncoding('utf8'); + +// This hangs for input, and process.exit ensures it only triggers one +process.stdin.on('data', function (affectedProjectsString) { + // If there is no input, nothing is affected. Bail early. + if (affectedProjectsString.trim().length === 0) { + console.log('Success: No projects affected'); + process.exit(0); + } + + // Handle multiple projects or a single project being affected + let affectedProjects = + affectedProjectsString.indexOf(',') !== -1 + ? affectedProjectsString.split(',') + : [affectedProjectsString.trim()]; + + let affectedProjectsCount = 0; + + affectedProjects.forEach((p) => { + const cleanedProjectName = p.trim(); + + // Path (from cwd) for the project, used for execSync if it's a fleek project + const project = `${projectPath}${cleanedProjectName}/`; + + // Specific file to check for the existence of + const fleekFilePath = `${project}${fleekFile}`; + + if (existsSync(fleekFilePath)) { + affectedProjectsCount++; + + console.group(`${cleanedProjectName} requires deployment`); + + // This will throw if it fails to trigger + execSync(deployCommand, { cwd: project }); + + console.groupEnd(); + } + }); + + // If we made it here, either we didn't have any projects to deploy... + if (affectedProjectsCount === 0) { + console.log('Success: No Fleek projects affected'); + } else { + // ... or all the projects deployed successfully... + console.log(`Success: ${affectedProjectsCount} deployments triggered`); + } + // ... so either way it's considered success + process.exit(0); +}); From 32cef6ef0c0dfa6dce1da08297e72d143a55ed05 Mon Sep 17 00:00:00 2001 From: Edd Date: Mon, 11 Apr 2022 12:42:22 +0100 Subject: [PATCH 03/10] Genericise tool name, add workflow --- .github/workflows/ipfs.yml | 33 +++++++++++++++++++++++ tools/{fleek-deploy.js => ipfs-deploy.js} | 2 -- 2 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/ipfs.yml rename tools/{fleek-deploy.js => ipfs-deploy.js} (99%) diff --git a/.github/workflows/ipfs.yml b/.github/workflows/ipfs.yml new file mode 100644 index 000000000..a7d557256 --- /dev/null +++ b/.github/workflows/ipfs.yml @@ -0,0 +1,33 @@ +--- +name: Deploy affected projects to IPFS + +'on': + # Triggers the workflow on push to main branch + push: + branches: [master] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: null + +jobs: + master: + name: Publish NX Affected + runs-on: ubuntu-latest + if: ${{ github.event_name != 'pull_request' }} + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.ref }} + fetch-depth: 0 + - name: Derive appropriate SHAs for base and head for `nx affected` commands + uses: nrwl/nx-set-shas@v2 + with: + main-branch-name: master + - name: Use Node.js 16 + id: Node + uses: actions/setup-node@v2 + with: + node-version: 16.14.0 + - name: Run deploy for affected sites + run: yarn nx print-affected --select=projects | ./tools/ipfs-deploy.js diff --git a/tools/fleek-deploy.js b/tools/ipfs-deploy.js similarity index 99% rename from tools/fleek-deploy.js rename to tools/ipfs-deploy.js index 3817f23c4..83a8e606f 100755 --- a/tools/fleek-deploy.js +++ b/tools/ipfs-deploy.js @@ -3,8 +3,6 @@ /** * Runs the fleek deploy process based on nx:affected, but only for sites * that have a .fleek.json file - * - * @author Edd */ // Fleek CLI requires this variable to be set From 62944b89cd62061a7f0244a123cdb4399894b5a2 Mon Sep 17 00:00:00 2001 From: Edd Date: Mon, 11 Apr 2022 14:45:45 +0100 Subject: [PATCH 04/10] Remove redundant check Co-authored-by: Dexter Edwards --- .github/workflows/ipfs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ipfs.yml b/.github/workflows/ipfs.yml index a7d557256..15bbac8a3 100644 --- a/.github/workflows/ipfs.yml +++ b/.github/workflows/ipfs.yml @@ -13,7 +13,6 @@ jobs: master: name: Publish NX Affected runs-on: ubuntu-latest - if: ${{ github.event_name != 'pull_request' }} steps: - name: Checkout uses: actions/checkout@v2 From 0e13248bee337da2745c19c7b30335d043cf549c Mon Sep 17 00:00:00 2001 From: Edd Date: Mon, 11 Apr 2022 14:54:10 +0100 Subject: [PATCH 05/10] Address review comments --- .github/workflows/ipfs.yml | 4 ---- tools/ipfs-deploy.js | 5 ++--- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ipfs.yml b/.github/workflows/ipfs.yml index 15bbac8a3..7ea3c044c 100644 --- a/.github/workflows/ipfs.yml +++ b/.github/workflows/ipfs.yml @@ -19,10 +19,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.ref }} fetch-depth: 0 - - name: Derive appropriate SHAs for base and head for `nx affected` commands - uses: nrwl/nx-set-shas@v2 - with: - main-branch-name: master - name: Use Node.js 16 id: Node uses: actions/setup-node@v2 diff --git a/tools/ipfs-deploy.js b/tools/ipfs-deploy.js index 83a8e606f..a137428e9 100755 --- a/tools/ipfs-deploy.js +++ b/tools/ipfs-deploy.js @@ -4,6 +4,8 @@ * Runs the fleek deploy process based on nx:affected, but only for sites * that have a .fleek.json file */ +const { existsSync } = require('fs'); +const { execSync } = require('child_process'); // Fleek CLI requires this variable to be set if (!process.env['FLEEK_API_KEY']) { @@ -11,9 +13,6 @@ if (!process.env['FLEEK_API_KEY']) { process.exit(1); } -const { existsSync } = require('fs'); -const { execSync } = require('child_process'); - // The folder containing NX projects const projectPath = './apps/'; // The Fleek project file, the existence indicates a deployed app From 079d09e3d928d9f38adff83d0fa6ae6a880c90ad Mon Sep 17 00:00:00 2001 From: Edd Date: Mon, 11 Apr 2022 14:56:55 +0100 Subject: [PATCH 06/10] Hardcode commits to last master commit --- .github/workflows/ipfs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ipfs.yml b/.github/workflows/ipfs.yml index 7ea3c044c..20f3c8d14 100644 --- a/.github/workflows/ipfs.yml +++ b/.github/workflows/ipfs.yml @@ -25,4 +25,4 @@ jobs: with: node-version: 16.14.0 - name: Run deploy for affected sites - run: yarn nx print-affected --select=projects | ./tools/ipfs-deploy.js + run: yarn nx print-affected --base=origin/master~1 --head=origin/master --select=projects | ./tools/ipfs-deploy.js From bfdb1e0b6245d114bde031e29833b2e673a281ae Mon Sep 17 00:00:00 2001 From: Edd Date: Mon, 11 Apr 2022 18:51:32 +0100 Subject: [PATCH 07/10] Rewrite ipfs-deploy to work differently Feedback on the initial PR pointed out the flaw that just assuming that the previous commit was HEAD~1 would be inaccurate, so this rewrite instead fetches the appropriate commit for each Fleek project, does the appropriate nx affected call, and triggers a deploy. This allows for each project to independently get deployed only when there is a change that affects it on master. The script is written to have no dependencies, and lazily uses curl instead of node http. It feels a bit icky, but for a build script feels reasonable. Also as the header comment notes, it seems like an nx build script would be more appropriate, but because it needs the appropriate commit for each project, I resorted to a shell script. - Rewrite ./tools/ipfs-deploy.js --- tools/ipfs-deploy.js | 181 +++++++++++++++++++++++++++++-------------- 1 file changed, 124 insertions(+), 57 deletions(-) diff --git a/tools/ipfs-deploy.js b/tools/ipfs-deploy.js index a137428e9..02ea2d613 100755 --- a/tools/ipfs-deploy.js +++ b/tools/ipfs-deploy.js @@ -1,73 +1,140 @@ #!/usr/bin/env node /** - * Runs the fleek deploy process based on nx:affected, but only for sites - * that have a .fleek.json file + * Run on Github by commits to master, this script: + * 1. Gets all projects with a fleek configuration file + * 2. Gets the last commit for the relevant site id + * 3. Runs nx:affected for that commit ID and checks if the site has changed + * 4. Calls deploy if it has + * + * This would probably be best as an NX task, but the circular nature of getting + * the fleek ID for the project, then checking if it was affected didn't fit within the + * build-only-affected cycle, and as each fleek deploy will have been triggered by + * a different commit, it seemed best to do this outwith nx. Feel free to re-implement + * this if the assumptions are wrong. + * + * It has also been written to skip external dependencies. */ -const { existsSync } = require('fs'); +const { existsSync, readdirSync, lstatSync, readFileSync } = require('fs'); const { execSync } = require('child_process'); +/** + * Parses the last commit hash out of the Fleek API response + * @param {String} siteId + * @returns string Last commit that triggered a build + */ +function getFleekLastBuildCommit(siteId) { + const curl = `curl 'https://api.fleek.co/graphql' --silent -X POST -H 'Accept: */*' -H 'Accept-Encoding: gzip, deflate, br' -H 'Content-Type: application/json' -H 'Authorization: ${process.env['FLEEK_API_KEY']}' --data-raw '{"query":"{getSiteById(siteId: \\"${siteId}\\"){publishedDeploy{repository{commit}}}}","variables":null}'`; + const fleekRes = execSync(curl); + + const res = JSON.parse(fleekRes.toString()); + let commit = res.data.getSiteById.publishedDeploy.repository.commit; + + return commit; +} + +function triggerDeploy(siteId) { + const curl = `curl 'https://api.fleek.co/graphql' --silent -X POST -H 'Accept: */*' -H 'Accept-Encoding: gzip, deflate, br' -H 'Content-Type: application/json' -H 'Authorization: ${process.env['FLEEK_API_KEY']}' --data-raw '{"query":"mutation {triggerDeploy(commit: \\"HEAD\\", siteId: \\"${siteId}\\"){status}}","variables":null}'`; + const fleekRes = execSync(curl); + + const res = JSON.parse(fleekRes.toString()); + + // Will have thrown if it failed + return true; +} + +// The folder containing NX projects +const projectPath = './apps/'; +// The Fleek project file, the existence indicates a deployed app +const fleekFile = '.fleek.json'; +// Some simple stats for the end +let fleekProjects = 0; +let deployedProjects = 0; + // Fleek CLI requires this variable to be set if (!process.env['FLEEK_API_KEY']) { console.error('Error: FLEEK_API_KEY must be set'); process.exit(1); } -// The folder containing NX projects -const projectPath = './apps/'; -// The Fleek project file, the existence indicates a deployed app -const fleekFile = '.fleek.json'; -// Command to run in each app that needs to be deployed -const deployCommand = 'npx @fleekhq/fleek-cli@0.1.8 site:deploy'; - -// Await piped input -process.stdin.resume(); -process.stdin.setEncoding('utf8'); - -// This hangs for input, and process.exit ensures it only triggers one -process.stdin.on('data', function (affectedProjectsString) { - // If there is no input, nothing is affected. Bail early. - if (affectedProjectsString.trim().length === 0) { - console.log('Success: No projects affected'); - process.exit(0); - } - - // Handle multiple projects or a single project being affected - let affectedProjects = - affectedProjectsString.indexOf(',') !== -1 - ? affectedProjectsString.split(',') - : [affectedProjectsString.trim()]; - - let affectedProjectsCount = 0; - - affectedProjects.forEach((p) => { - const cleanedProjectName = p.trim(); - - // Path (from cwd) for the project, used for execSync if it's a fleek project - const project = `${projectPath}${cleanedProjectName}/`; - - // Specific file to check for the existence of - const fleekFilePath = `${project}${fleekFile}`; - - if (existsSync(fleekFilePath)) { - affectedProjectsCount++; - - console.group(`${cleanedProjectName} requires deployment`); - - // This will throw if it fails to trigger - execSync(deployCommand, { cwd: project }); - - console.groupEnd(); +readdirSync(projectPath).forEach((proj) => { + try { + const project = `${projectPath}${proj}`; + if (!lstatSync(project).isDirectory()) { + // It's not a project folder, skip it + return; } - }); - // If we made it here, either we didn't have any projects to deploy... - if (affectedProjectsCount === 0) { - console.log('Success: No Fleek projects affected'); - } else { - // ... or all the projects deployed successfully... - console.log(`Success: ${affectedProjectsCount} deployments triggered`); + const config = `${project}/${fleekFile}`; + if (!existsSync(config)) { + // No fleek file, skip it + return; + } + + fleekProjects++; + + console.group(proj); + + // The UID for the site according to the config + let siteId; + + try { + const fleekConfig = JSON.parse(readFileSync(config)); + siteId = fleekConfig.site.id; + + console.log(`Fleek site ID: ${siteId}`); + } catch (e) { + console.error(`Failed to read Fleek site id for ${proj}`); + return; + } + + // The last commit that triggered a build + let baseCommit; + + try { + baseCommit = getFleekLastBuildCommit(siteId); + + console.log(`Last deploy: ${baseCommit}`); + } catch (e) { + console.error(`Failed to fetch last deploy for ${proj}`); + return; + } + + // Now run nx affected + let isAffected; + + try { + const affectedSinceCommit = execSync( + `yarn nx print-affected --base=${baseCommit} --head=master --select=projects` + ); + isAffected = affectedSinceCommit.toString().indexOf(proj) !== -1; + } catch (e) { + console.error(`Failed to run nx:affected for ${baseCommit}:master`); + return; + } + + if (isAffected) { + console.log(`Triggering deploy for: ${siteId}`); + deployedProjects++; + + try { + triggerDeploy(siteId); + } catch (e) { + console.error(`Failed to trigger deploy for ${proj}`); + process.exit(1); + } + } else { + console.log(`Has not changed since last build, skipping...`); + } + + console.groupEnd(); + } catch (e) { + console.log(e); + process.exit(1); } - // ... so either way it's considered success - process.exit(0); }); + +console.log(`Fleek projects: ${fleekProjects}`); +console.log(`Deploys triggered: ${deployedProjects}`); + +process.exit(0); From 47f1f7358290e810f9a7607df757b700170a7eb5 Mon Sep 17 00:00:00 2001 From: Edd Date: Mon, 11 Apr 2022 18:55:59 +0100 Subject: [PATCH 08/10] Update workflow to remove pre-script nx affected --- .github/workflows/ipfs.yml | 2 +- tools/ipfs-deploy.js | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ipfs.yml b/.github/workflows/ipfs.yml index 20f3c8d14..524f8f4f6 100644 --- a/.github/workflows/ipfs.yml +++ b/.github/workflows/ipfs.yml @@ -25,4 +25,4 @@ jobs: with: node-version: 16.14.0 - name: Run deploy for affected sites - run: yarn nx print-affected --base=origin/master~1 --head=origin/master --select=projects | ./tools/ipfs-deploy.js + run: ./tools/ipfs-deploy.js diff --git a/tools/ipfs-deploy.js b/tools/ipfs-deploy.js index 02ea2d613..07e1f055a 100755 --- a/tools/ipfs-deploy.js +++ b/tools/ipfs-deploy.js @@ -33,6 +33,11 @@ function getFleekLastBuildCommit(siteId) { return commit; } +/** + * Triggers a Fleek build of the latest code via GraphQL + * @param {String} siteId + * @returns + */ function triggerDeploy(siteId) { const curl = `curl 'https://api.fleek.co/graphql' --silent -X POST -H 'Accept: */*' -H 'Accept-Encoding: gzip, deflate, br' -H 'Content-Type: application/json' -H 'Authorization: ${process.env['FLEEK_API_KEY']}' --data-raw '{"query":"mutation {triggerDeploy(commit: \\"HEAD\\", siteId: \\"${siteId}\\"){status}}","variables":null}'`; const fleekRes = execSync(curl); From d1e50208aa50db50d5173ce411093eafaed33ca5 Mon Sep 17 00:00:00 2001 From: Edd Date: Tue, 12 Apr 2022 10:48:15 +0100 Subject: [PATCH 09/10] Review feedback: remove redundant check --- tools/ipfs-deploy.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ipfs-deploy.js b/tools/ipfs-deploy.js index 07e1f055a..eb0bec7d9 100755 --- a/tools/ipfs-deploy.js +++ b/tools/ipfs-deploy.js @@ -15,7 +15,7 @@ * * It has also been written to skip external dependencies. */ -const { existsSync, readdirSync, lstatSync, readFileSync } = require('fs'); +const { existsSync, readdirSync, readFileSync } = require('fs'); const { execSync } = require('child_process'); /** From ad4c7d88ff814614f8bf04ac37151d70f8e7445d Mon Sep 17 00:00:00 2001 From: Edd Date: Tue, 12 Apr 2022 10:52:10 +0100 Subject: [PATCH 10/10] Review feedback: better affected detection that avoids collisions --- tools/ipfs-deploy.js | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tools/ipfs-deploy.js b/tools/ipfs-deploy.js index eb0bec7d9..6fbf67617 100755 --- a/tools/ipfs-deploy.js +++ b/tools/ipfs-deploy.js @@ -42,7 +42,7 @@ function triggerDeploy(siteId) { const curl = `curl 'https://api.fleek.co/graphql' --silent -X POST -H 'Accept: */*' -H 'Accept-Encoding: gzip, deflate, br' -H 'Content-Type: application/json' -H 'Authorization: ${process.env['FLEEK_API_KEY']}' --data-raw '{"query":"mutation {triggerDeploy(commit: \\"HEAD\\", siteId: \\"${siteId}\\"){status}}","variables":null}'`; const fleekRes = execSync(curl); - const res = JSON.parse(fleekRes.toString()); + JSON.parse(fleekRes.toString()); // Will have thrown if it failed return true; @@ -64,13 +64,7 @@ if (!process.env['FLEEK_API_KEY']) { readdirSync(projectPath).forEach((proj) => { try { - const project = `${projectPath}${proj}`; - if (!lstatSync(project).isDirectory()) { - // It's not a project folder, skip it - return; - } - - const config = `${project}/${fleekFile}`; + const config = `${projectPath}${proj}/${fleekFile}`; if (!existsSync(config)) { // No fleek file, skip it return; @@ -112,7 +106,14 @@ readdirSync(projectPath).forEach((proj) => { const affectedSinceCommit = execSync( `yarn nx print-affected --base=${baseCommit} --head=master --select=projects` ); - isAffected = affectedSinceCommit.toString().indexOf(proj) !== -1; + + // Detect if this project name is in output, taking care not to match names that are + // included in other projects - `trading`, `trading-e2e` is a current example + isAffected = affectedSinceCommit + .toString() + .split(',') + .map((v) => v.trim()) + .includes(proj); } catch (e) { console.error(`Failed to run nx:affected for ${baseCommit}:master`); return;