Create deployments on push events in GitHub repo (#69)

* Create repo webhook and express handler for webhook

* Create deployments from commits in GitHub

* Update isCurrent in previous production deployment

* Create script for setting authority

* Update README for initialize registry script

* Handle review changes

* Use correct repo URL in record data

* Handle github unique webhook error

* Handle async execution of publishing records

* Update readme with ngrok setup

* Review changes

* Add logs for GitHub webhooks

---------

Co-authored-by: neeraj <neeraj.rtly@gmail.com>
This commit is contained in:
Nabarun Gogoi 2024-02-15 17:24:57 +05:30 committed by GitHub
parent 9921dc5186
commit c4ba59d97e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
19 changed files with 289 additions and 102 deletions

View File

@ -38,25 +38,11 @@
- Client ID and secret will be available after creating Github OAuth app
- https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/creating-an-oauth-app
- In "Homepage URL", type `http://localhost:3000`
- In "Authorization callback URL", type `http://localhost:3000/projects/create`
- In "Authorization callback URL", type `http://localhost:3000/organization/projects/create`
- Generate a new client secret after app is created
- Run the laconicd stack following this [doc](https://git.vdb.to/cerc-io/stack-orchestrator/src/branch/main/docs/laconicd-with-console.md)
- Create the bond and set `registryConfig.bondId` in backend [config file](packages/backend/environments/local.toml)
```bash
laconic-so --stack fixturenet-laconic-loaded deploy exec cli "laconic cns bond create --type aphoton --quantity 1000000000 --gas 200000 --fees 200000aphoton"
# {"bondId":"b40f1308510f799860fb6f1ede47245a2d59f336631158f25ae0eec30aabaf89"}
```
- Export the bond id that is generated
```bash
export BOND_ID=<BOND-ID>
```
- Get the private key and set `registryConfig.privateKey` in backend [config file](packages/backend/environments/local.toml)
```bash
@ -65,7 +51,7 @@
# 754cca7b4b729a99d156913aea95366411d072856666e95ba09ef6c664357d81
```
- Get the rest and GQL endpoint of laconicd and set it to `registryConfig.restEndpoint` and `registryConfig.gqlEndpoint` in backend [config file](packages/backend/environments/local.toml)
- Get the REST and GQL endpoint ports of Laconicd and replace the ports for `registryConfig.restEndpoint` and `registryConfig.gqlEndpoint` in backend [config file](packages/backend/environments/local.toml)
```bash
# For registryConfig.restEndpoint
@ -77,28 +63,33 @@
# 0.0.0.0:32771
```
- Reserve authorities for `snowballtools` and `cerc-io`
- Run the script to create bond, reserve the authority and set authority bond
```bash
laconic-so --stack fixturenet-laconic-loaded deploy exec cli "laconic cns authority reserve snowballtools"
# {"success":true}
yarn registry:init
# snowball:initialize-registry bondId: 6af0ab81973b93d3511ae79841756fb5da3fd2f70ea1279e81fae7c9b19af6c4 +0ms
```
- Get the bond id and set `registryConfig.bondId` in backend [config file](packages/backend/environments/local.toml)
- Setup ngrok for GitHub webhooks
- https://ngrok.com/docs/getting-started/
- Start ngrok and point to backend server endpoint
```bash
laconic-so --stack fixturenet-laconic-loaded deploy exec cli "laconic cns authority reserve cerc-io"
# {"success":true}
ngrok http http://localhost:8000
```
- Set authority bond for `snowballtools` and `cerc-io`
```bash
laconic-so --stack fixturenet-laconic-loaded deploy exec cli "laconic cns authority bond set snowballtools $BOND_ID"
# {"success":true}
- Look for the forwarding URL in ngrok
```
```bash
laconic-so --stack fixturenet-laconic-loaded deploy exec cli "laconic cns authority bond set cerc-io $BOND_ID"
# {"success":true}
...
Forwarding https://19c1-61-95-158-116.ngrok-free.app -> http://localhost:8000
...
```
- Set `gitHub.webhookUrl` in backend [config file](packages/backend/environments/local.toml)
```toml
...
[gitHub]
webhookUrl = "https://19c1-61-95-158-116.ngrok-free.app"
...
```
- Start the server in `packages/backend`
@ -127,6 +118,12 @@
REACT_APP_GITHUB_CLIENT_ID = <CLIENT_ID>
```
- Set `REACT_APP_GITHUB_TEMPLATE_REPO` in [.env](packages/frontend/.env) file
```env
REACT_APP_GITHUB_TEMPLATE_REPO = cerc-io/test-progressive-web-app
```
### Development
- Start the React application

View File

@ -6,7 +6,9 @@
[database]
dbPath = "db/snowball"
[githubOauth]
[gitHub]
webhookUrl = ""
[gitHub.oAuth]
clientId = ""
clientSecret = ""

View File

@ -36,6 +36,7 @@
"lint": "eslint .",
"format": "prettier --write .",
"format:check": "prettier --check .",
"registry:init": "DEBUG=snowball:* ts-node scripts/initialize-registry.ts",
"db:load:fixtures": "DEBUG=snowball:* ts-node ./test/initialize-db.ts",
"db:delete": "DEBUG=snowball:* ts-node ./test/delete-db.ts"
},

View File

@ -0,0 +1,36 @@
import debug from 'debug';
import { Registry } from '@cerc-io/laconic-sdk';
import { DEFAULT_CONFIG_FILE_PATH } from '../src/constants';
import { Config } from '../src/config';
import { getConfig } from '../src/utils';
const log = debug('snowball:initialize-registry');
const DENOM = 'aphoton';
const BOND_AMOUNT = '1000000000';
// TODO: Get authority names from args
const AUTHORITY_NAMES = ['snowballtools', 'cerc-io'];
async function main () {
const { registryConfig } = await getConfig<Config>(DEFAULT_CONFIG_FILE_PATH);
const registry = new Registry(registryConfig.gqlEndpoint, registryConfig.restEndpoint, registryConfig.chainId);
const bondId = await registry.getNextBondId(registryConfig.privateKey);
log('bondId:', bondId);
await registry.createBond({ denom: DENOM, amount: BOND_AMOUNT }, registryConfig.privateKey, registryConfig.fee);
for await (const name of AUTHORITY_NAMES) {
await registry.reserveAuthority({ name }, registryConfig.privateKey, registryConfig.fee);
log('Reserved authority name:', name);
await registry.setAuthorityBond({ name, bondId }, registryConfig.privateKey, registryConfig.fee);
log(`Bond ${bondId} set for authority ${name}`);
}
}
main().catch((err) => {
log(err);
});

View File

@ -8,10 +8,13 @@ export interface DatabaseConfig {
dbPath: string;
}
export interface GithubOauthConfig {
export interface GitHubConfig {
webhookUrl: string;
oAuth: {
clientId: string;
clientSecret: string;
}
}
export interface RegistryConfig {
restEndpoint: string;
@ -29,6 +32,6 @@ export interface RegistryConfig {
export interface Config {
server: ServerConfig;
database: DatabaseConfig;
githubOauth: GithubOauthConfig;
gitHub: GitHubConfig;
registryConfig: RegistryConfig;
}

View File

@ -83,6 +83,13 @@ export class Database {
return userOrgs;
}
async getProjects (options: FindManyOptions<Project>): Promise<Project[]> {
const projectRepository = this.dataSource.getRepository(Project);
const projects = await projectRepository.find(options);
return projects;
}
async getProjectById (projectId: string): Promise<Project | null> {
const projectRepository = this.dataSource.getRepository(Project);
@ -294,8 +301,12 @@ export class Database {
}
async updateDeploymentById (deploymentId: string, data: DeepPartial<Deployment>): Promise<boolean> {
return this.updateDeployment({ id: deploymentId }, data);
}
async updateDeployment (criteria: FindOptionsWhere<Deployment>, data: DeepPartial<Deployment>): Promise<boolean> {
const deploymentRepository = this.dataSource.getRepository(Deployment);
const updateResult = await deploymentRepository.update({ id: deploymentId }, data);
const updateResult = await deploymentRepository.update(criteria, data);
return Boolean(updateResult.affected);
}

View File

@ -49,6 +49,9 @@ export class Deployment {
@JoinColumn({ name: 'projectId' })
project!: Project;
@Column({ nullable: true })
domainId!: string | null;
@OneToOne(() => Domain)
@JoinColumn({ name: 'domainId' })
domain!: Domain | null;

View File

@ -33,6 +33,9 @@ export class Project {
@JoinColumn({ name: 'ownerId' })
owner!: User;
@Column({ nullable: false })
ownerId!: string;
@ManyToOne(() => Organization, { nullable: true })
@JoinColumn({ name: 'organizationId' })
organization!: Organization | null;

View File

@ -19,24 +19,24 @@ const OAUTH_CLIENT_TYPE = 'oauth-app';
export const main = async (): Promise<void> => {
// TODO: get config path using cli
const { server, database, githubOauth, registryConfig } = await getConfig<Config>(DEFAULT_CONFIG_FILE_PATH);
const { server, database, gitHub, registryConfig } = await getConfig<Config>(DEFAULT_CONFIG_FILE_PATH);
const app = new OAuthApp({
clientType: OAUTH_CLIENT_TYPE,
clientId: githubOauth.clientId,
clientSecret: githubOauth.clientSecret
clientId: gitHub.oAuth.clientId,
clientSecret: gitHub.oAuth.clientSecret
});
const db = new Database(database);
await db.init();
const registry = new Registry(registryConfig);
const service = new Service(db, app, registry);
const service = new Service({ gitHubConfig: gitHub }, db, app, registry);
const typeDefs = fs.readFileSync(path.join(__dirname, 'schema.gql')).toString();
const resolvers = await createResolvers(service);
await createAndStartServer(typeDefs, resolvers, server);
await createAndStartServer(server, typeDefs, resolvers, service);
};
main()

View File

@ -28,11 +28,13 @@ export class Registry {
async createApplicationRecord ({
packageJSON,
commitHash,
appType
appType,
repoUrl
}: {
packageJSON: PackageJSON
commitHash: string,
appType: string,
commitHash: string
repoUrl: string
}): Promise<{registryRecordId: string, registryRecordData: ApplicationRecord}> {
// Use laconic-sdk to publish record
// Reference: https://git.vdb.to/cerc-io/test-progressive-web-app/src/branch/main/scripts/publish-app-record.sh
@ -54,13 +56,13 @@ export class Registry {
type: APP_RECORD_TYPE,
version: nextVersion,
repository_ref: commitHash,
repository: [repoUrl],
app_type: appType,
...(packageJSON.name && { name: packageJSON.name }),
...(packageJSON.description && { description: packageJSON.description }),
...(packageJSON.homepage && { homepage: packageJSON.homepage }),
...(packageJSON.license && { license: packageJSON.license }),
...(packageJSON.author && { author: typeof packageJSON.author === 'object' ? JSON.stringify(packageJSON.author) : packageJSON.author }),
...(packageJSON.repository && { repository: [packageJSON.repository] }),
...(packageJSON.version && { app_version: packageJSON.version })
};
@ -78,6 +80,7 @@ export class Registry {
// TODO: Discuss computation of CRN
const crn = this.getCrn(packageJSON.name ?? '');
log(`Setting name: ${crn} for record ID: ${result.data.id}`);
await this.registry.setName({ cid: result.data.id, crn }, this.registryConfig.privateKey, this.registryConfig.fee);
await this.registry.setName({ cid: result.data.id, crn: `${crn}@${applicationRecord.app_version}` }, this.registryConfig.privateKey, this.registryConfig.fee);

View File

@ -0,0 +1,26 @@
import { Router } from 'express';
import debug from 'debug';
import { Service } from '../service';
const log = debug('snowball:routes-github');
const router = Router();
/* POST GitHub webhook handler */
// https://docs.github.com/en/webhooks/using-webhooks/handling-webhook-deliveries#javascript-example
router.post('/webhook', async (req, res) => {
// Server should respond with a 2XX response within 10 seconds of receiving a webhook delivery
// If server takes longer than that to respond, then GitHub terminates the connection and considers the delivery a failure
res.status(202).send('Accepted');
const service = req.app.get('service') as Service;
const githubEvent = req.headers['x-github-event'];
log(`Received GitHub webhook for event ${githubEvent}`);
if (githubEvent === 'push') {
// Create deployments using push event data
await service.handleGitHubPush(req.body);
}
});
export default router;

View File

@ -12,13 +12,16 @@ import { makeExecutableSchema } from '@graphql-tools/schema';
import { ServerConfig } from './config';
import { DEFAULT_GQL_PATH, USER_ID } from './constants';
import githubRouter from './routes/github';
import { Service } from './service';
const log = debug('snowball:server');
export const createAndStartServer = async (
serverConfig: ServerConfig,
typeDefs: TypeSource,
resolvers: any,
serverConfig: ServerConfig
service: Service
): Promise<ApolloServer> => {
const { host, port, gqlPath = DEFAULT_GQL_PATH } = serverConfig;
@ -54,6 +57,10 @@ export const createAndStartServer = async (
path: gqlPath
});
app.set('service', service);
app.use(express.json());
app.use('/api/github', githubRouter);
httpServer.listen(port, host, () => {
log(`Server is listening on ${host}:${port}${server.graphqlPath}`);
});

View File

@ -1,7 +1,7 @@
import assert from 'assert';
import debug from 'debug';
import { DeepPartial, FindOptionsWhere } from 'typeorm';
import { Octokit } from 'octokit';
import { Octokit, RequestError } from 'octokit';
import { OAuthApp } from '@octokit/oauth-app';
@ -14,18 +14,27 @@ import { Project } from './entity/Project';
import { Permission, ProjectMember } from './entity/ProjectMember';
import { User } from './entity/User';
import { Registry } from './registry';
import { GitHubConfig } from './config';
import { GitPushEventPayload } from './types';
const log = debug('snowball:service');
const GITHUB_UNIQUE_WEBHOOK_ERROR = 'Hook already exists on this repository';
interface Config {
gitHubConfig: GitHubConfig
}
export class Service {
private db: Database;
private oauthApp: OAuthApp;
private registry: Registry;
private config: Config;
constructor (db: Database, app: OAuthApp, registry: Registry) {
constructor (config: Config, db: Database, app: OAuthApp, registry: Registry) {
this.db = db;
this.oauthApp = app;
this.registry = registry;
this.config = config;
}
async getUser (userId: string): Promise<User | null> {
@ -176,38 +185,6 @@ export class Service {
const prodBranchDomains = await this.db.getDomainsByProjectId(oldDeployment.project.id, { branch: oldDeployment.project.prodBranch });
// TODO: Fix unique constraint error for domain
const deploymentWithProdBranchDomain = await this.db.getDeployment({
where: {
domain: {
id: prodBranchDomains[0].id
}
}
});
if (deploymentWithProdBranchDomain) {
await this.db.updateDeploymentById(deploymentWithProdBranchDomain.id, {
domain: null,
isCurrent: false
});
}
const oldCurrentDeployment = await this.db.getDeployment({
relations: {
domain: true
},
where: {
project: {
id: oldDeployment.project.id
},
isCurrent: true
}
});
if (oldCurrentDeployment) {
await this.db.updateDeploymentById(oldCurrentDeployment.id, { isCurrent: false, domain: null });
}
const octokit = await this.getOctokit(userId);
const newDeployement = await this.createDeployment(userId,
@ -225,8 +202,14 @@ export class Service {
return newDeployement;
}
async createDeployment (userId: string, octokit: Octokit, data: DeepPartial<Deployment>): Promise<Deployment> {
async createDeployment (
userId: string,
octokit: Octokit,
data: DeepPartial<Deployment>,
recordData: { repoUrl?: string } = {}
): Promise<Deployment> {
assert(data.project?.repository, 'Project repository not found');
log(`Creating deployment in project ${data.project.name} from branch ${data.branch}`);
const [owner, repo] = data.project.repository.split('/');
const { data: packageJSONData } = await octokit.rest.repos.getContent({
@ -243,10 +226,35 @@ export class Service {
assert(!Array.isArray(packageJSONData) && packageJSONData.type === 'file');
const packageJSON = JSON.parse(atob(packageJSONData.content));
if (!recordData.repoUrl) {
const { data: repoDetails } = await octokit.rest.repos.get({ owner, repo });
recordData.repoUrl = repoDetails.html_url;
}
const { registryRecordId, registryRecordData } = await this.registry.createApplicationRecord({
packageJSON,
appType: data.project!.template!,
commitHash: data.commitHash!
commitHash: data.commitHash!,
repoUrl: recordData.repoUrl
});
// Check if new deployment is set to current
if (data.isCurrent) {
// Update previous current deployment
await this.db.updateDeployment({
project: {
id: data.project.id
},
isCurrent: true
}, { isCurrent: false });
}
// Update previous deployment with prod branch domain
// TODO: Fix unique constraint error for domain
await this.db.updateDeployment({
domainId: data.domain?.id
}, {
domain: null
});
const newDeployement = await this.db.addDeployement({
@ -265,7 +273,7 @@ export class Service {
})
});
log(`Application record ${registryRecordId} published for deployment ${newDeployement.id}`);
log(`Created deployment ${newDeployement.id} and published application record ${registryRecordId}`);
return newDeployement;
}
@ -304,29 +312,91 @@ export class Service {
domain: null,
commitHash: latestCommit.sha,
commitMessage: latestCommit.commit.message
});
},
{
repoUrl: repoDetails.html_url
}
);
const { registryRecordId, registryRecordData } = await this.registry.createApplicationDeploymentRequest(
{
appName: newDeployment.registryRecordData.name!,
commitHash: latestCommit.sha,
repository: repoDetails.git_url
repository: repoDetails.html_url
});
await this.db.updateProjectById(project.id, {
registryRecordId,
registryRecordData
});
// TODO: Setup repo webhook for push events
await this.createRepoHook(octokit, project);
return project;
}
async createRepoHook (octokit: Octokit, project: Project): Promise<void> {
try {
const [owner, repo] = project.repository.split('/');
await octokit.rest.repos.createWebhook({
owner,
repo,
config: {
url: new URL('api/github/webhook', this.config.gitHubConfig.webhookUrl).href,
content_type: 'json'
},
events: ['push']
});
} catch (err) {
// https://docs.github.com/en/rest/repos/webhooks?apiVersion=2022-11-28#create-a-repository-webhook--status-codes
if (
!(err instanceof RequestError &&
err.status === 422 &&
(err.response?.data as any).errors.some((err: any) => err.message === GITHUB_UNIQUE_WEBHOOK_ERROR))) {
throw err;
}
log(GITHUB_UNIQUE_WEBHOOK_ERROR);
}
}
async handleGitHubPush (data: GitPushEventPayload): Promise<void> {
const { repository, ref, head_commit: headCommit } = data;
log(`Handling GitHub push event from repository: ${repository.full_name}`);
const projects = await this.db.getProjects({ where: { repository: repository.full_name } });
if (!projects.length) {
log(`No projects found for repository ${repository.full_name}`);
}
// The `ref` property contains the full reference, including the branch name
// For example, "refs/heads/main" or "refs/heads/feature-branch"
const branch = ref.split('/').pop();
for await (const project of projects) {
const octokit = await this.getOctokit(project.ownerId);
const [domain] = await this.db.getDomainsByProjectId(project.id, { branch });
// Create deployment with branch and latest commit in GitHub data
await this.createDeployment(project.ownerId,
octokit,
{
project,
isCurrent: project.prodBranch === branch,
branch,
environment: project.prodBranch === branch ? Environment.Production : Environment.Preview,
domain,
commitHash: headCommit.id,
commitMessage: headCommit.message
});
}
}
async updateProject (projectId: string, data: DeepPartial<Project>): Promise<boolean> {
return this.db.updateProjectById(projectId, data);
}
async deleteProject (projectId: string): Promise<boolean> {
// TODO: Remove GitHub repo hook
return this.db.deleteProjectById(projectId);
}
@ -360,8 +430,6 @@ export class Service {
throw new Error('Deployment not found');
}
await this.db.updateDeploymentById(deploymentId, { domain: null, isCurrent: false });
const octokit = await this.getOctokit(userId);
const newDeployement = await this.createDeployment(userId,

View File

@ -7,3 +7,21 @@ export interface PackageJSON {
license?: string;
repository?: string;
}
export interface GitRepositoryDetails {
id: number;
name: string;
full_name: string;
visibility?: string;
updated_at?: string | null;
default_branch?: string;
}
export interface GitPushEventPayload {
repository: GitRepositoryDetails;
ref: string;
head_commit: {
id: string;
message: string;
};
}

View File

@ -1,3 +1,4 @@
REACT_APP_GQL_SERVER_URL = 'http://localhost:8000/graphql'
REACT_APP_GITHUB_CLIENT_ID =
REACT_APP_GITHUB_TEMPLATE_REPO =

View File

@ -3,10 +3,10 @@ import React from 'react';
import { Typography, IconButton } from '@material-tailwind/react';
import { relativeTimeISO } from '../../../utils/time';
import { GitCommitDetails } from '../../../types';
import { GitCommitWithBranch } from '../../../types';
interface ActivityCardProps {
activity: GitCommitDetails;
activity: GitCommitWithBranch;
}
const ActivityCard = ({ activity }: ActivityCardProps) => {

View File

@ -33,13 +33,19 @@ const CreateRepo = () => {
assert(data.account);
try {
assert(
process.env.REACT_APP_GITHUB_TEMPLATE_REPO,
'Config REACT_APP_GITHUB_TEMPLATE_REPO is not set in .env',
);
const [owner, repo] =
process.env.REACT_APP_GITHUB_TEMPLATE_REPO.split('/');
// TODO: Handle this functionality in backend
const gitRepo = await octokit?.rest.repos.createUsingTemplate({
template_owner: 'github-rest',
template_repo: 'test-progressive-web-app',
template_owner: owner,
template_repo: repo,
owner: data.account,
name: data.repoName,
description: 'This is your first repository',
include_all_branches: false,
private: data.isPrivate,
});
@ -60,6 +66,7 @@ const CreateRepo = () => {
`/${orgSlug}/projects/create/template/deploy?projectId=${addProject.id}`,
);
} catch (err) {
console.error(err);
toast.error('Error deploying project');
}
},

View File

@ -8,7 +8,7 @@ import { Typography, Button, Chip, Avatar } from '@material-tailwind/react';
import ActivityCard from '../../../../components/projects/project/ActivityCard';
import { relativeTimeMs } from '../../../../utils/time';
import { useOctokit } from '../../../../context/OctokitContext';
import { GitCommitDetails, OutletContextType } from '../../../../types';
import { GitCommitWithBranch, OutletContextType } from '../../../../types';
import { useGQLClient } from '../../../../context/GQLClientContext';
const COMMITS_PER_PAGE = 4;
@ -16,7 +16,7 @@ const COMMITS_PER_PAGE = 4;
const OverviewTabPanel = () => {
const { octokit } = useOctokit();
const navigate = useNavigate();
const [activities, setActivities] = useState<GitCommitDetails[]>([]);
const [activities, setActivities] = useState<GitCommitWithBranch[]>([]);
const [liveDomain, setLiveDomain] = useState<Domain>();
const client = useGQLClient();
@ -78,6 +78,7 @@ const OverviewTabPanel = () => {
throw err;
}
// TODO: Show warning in activity section on request error
console.log(err.message);
}
};

View File

@ -29,7 +29,7 @@ export interface GitBranchDetails {
name: string;
}
export interface GitCommitDetails {
export interface GitCommitWithBranch {
branch: GitBranchDetails;
commit: {
author: {