Merge branch 'ng-check-deployment-removal-record'

packages/backend/package.json

@@ -43,6 +43,7 @@
     "lint": "tsc --noEmit",
     "test:registry:init": "DEBUG=snowball:* ts-node ./test/initialize-registry.ts",
     "test:registry:publish-deploy-records": "DEBUG=snowball:* ts-node ./test/publish-deploy-records.ts",
+    "test:registry:publish-deployment-removal-records": "DEBUG=snowball:* ts-node ./test/publish-deployment-removal-records.ts",
     "test:db:load:fixtures": "DEBUG=snowball:* ts-node ./test/initialize-db.ts",
     "test:db:delete": "DEBUG=snowball:* ts-node ./test/delete-db.ts"
   },

packages/backend/src/database.ts

@@ -436,6 +436,19 @@ export class Database {
     return Boolean(updateResult.affected);
   }
 
+  async deleteDeploymentById (deploymentId: string): Promise<boolean> {
+    const deploymentRepository = this.dataSource.getRepository(Deployment);
+    const deployment = await deploymentRepository.findOneOrFail({
+      where: {
+        id: deploymentId
+      }
+    });
+
+    const deleteResult = await deploymentRepository.softRemove(deployment);
+
+    return Boolean(deleteResult);
+  }
+
   async addProject (user: User, organizationId: string, data: DeepPartial<Project>): Promise<Project> {
     const projectRepository = this.dataSource.getRepository(Project);
 

packages/backend/src/entity/Deployment.ts

@@ -6,13 +6,14 @@ import {
   UpdateDateColumn,
   ManyToOne,
   OneToOne,
-  JoinColumn
+  JoinColumn,
+  DeleteDateColumn
 } from 'typeorm';
 
 import { Project } from './Project';
 import { Domain } from './Domain';
 import { User } from './User';
-import { AppDeploymentRecordAttributes } from '../types';
+import { AppDeploymentRecordAttributes, AppDeploymentRemovalRecordAttributes } from '../types';
 
 export enum Environment {
   Production = 'Production',
@@ -24,6 +25,7 @@ export enum DeploymentStatus {
   Building = 'Building',
   Ready = 'Ready',
   Error = 'Error',
+  Deleting = 'Deleting',
 }
 
 export interface ApplicationDeploymentRequest {
@@ -35,6 +37,18 @@ export interface ApplicationDeploymentRequest {
   meta: string;
 }
 
+export interface ApplicationDeploymentRemovalRequest {
+  type: string;
+  version: string;
+  deployment: string;
+}
+
+export interface ApplicationDeploymentRemovalRequest {
+  type: string;
+  version: string;
+  deployment: string;
+}
+
 export interface ApplicationRecord {
   type: string;
   version: string;
@@ -98,6 +112,18 @@ export class Deployment {
 
   @Column('simple-json', { nullable: true })
     applicationDeploymentRecordData!: AppDeploymentRecordAttributes | null;
+  
+  @Column('varchar', { nullable: true })
+    applicationDeploymentRemovalRequestId!: string | null;
+
+  @Column('simple-json', { nullable: true })
+    applicationDeploymentRemovalRequestData!: ApplicationDeploymentRemovalRequest | null;
+  
+  @Column('varchar', { nullable: true })
+    applicationDeploymentRemovalRecordId!: string | null;
+
+  @Column('simple-json', { nullable: true })
+    applicationDeploymentRemovalRecordData!: AppDeploymentRemovalRecordAttributes | null;
 
   @Column({
     enum: Environment
@@ -121,4 +147,7 @@ export class Deployment {
 
   @UpdateDateColumn()
     updatedAt!: Date;
+  
+  @DeleteDateColumn()
+    deletedAt!: Date | null;
 }

packages/backend/src/registry.ts

@@ -9,16 +9,19 @@ import { RegistryConfig } from './config';
 import {
   ApplicationRecord,
   Deployment,
-  ApplicationDeploymentRequest
+  ApplicationDeploymentRequest,
+  ApplicationDeploymentRemovalRequest
 } from './entity/Deployment';
-import { AppDeploymentRecord, PackageJSON } from './types';
+import { AppDeploymentRecord, AppDeploymentRemovalRecord, PackageJSON } from './types';
 import { sleep } from './utils';
 
 const log = debug('snowball:registry');
 
 const APP_RECORD_TYPE = 'ApplicationRecord';
 const APP_DEPLOYMENT_REQUEST_TYPE = 'ApplicationDeploymentRequest';
+const APP_DEPLOYMENT_REMOVAL_REQUEST_TYPE = 'ApplicationDeploymentRemovalRequest';
 const APP_DEPLOYMENT_RECORD_TYPE = 'ApplicationDeploymentRecord';
+const APP_DEPLOYMENT_REMOVAL_RECORD_TYPE = 'ApplicationDeploymentRemovalRecord';
 const SLEEP_DURATION = 1000;
 
 // TODO: Move registry code to laconic-sdk/watcher-ts
@@ -229,6 +232,74 @@ export class Registry {
     );
   }
 
+  /**
+   * Fetch ApplicationDeploymentRecords by filter
+   */
+  async getDeploymentRecordsByFilter (filter: { [key: string]: any }): Promise<AppDeploymentRecord[]> {
+    return this.registry.queryRecords(
+      {
+        type: APP_DEPLOYMENT_RECORD_TYPE,
+        ...filter
+      },
+      true
+    );
+  }
+
+  /**
+   * Fetch ApplicationDeploymentRemovalRecords for deployments
+   */
+  async getDeploymentRemovalRecords (
+    deployments: Deployment[]
+  ): Promise<AppDeploymentRemovalRecord[]> {
+    // Fetch ApplicationDeploymentRemovalRecords for corresponding ApplicationDeploymentRecord set in deployments
+    const records = await this.registry.queryRecords(
+      {
+        type: APP_DEPLOYMENT_REMOVAL_RECORD_TYPE
+      },
+      true
+    );
+
+    // Filter records with ApplicationDeploymentRecord and ApplicationDeploymentRemovalRequest IDs
+    return records.filter((record: AppDeploymentRemovalRecord) =>
+      deployments.some(
+        (deployment) =>
+          deployment.applicationDeploymentRemovalRequestId === record.attributes.request &&
+          deployment.applicationDeploymentRecordId === record.attributes.deployment
+      )
+    );
+  }
+
+  async createApplicationDeploymentRemovalRequest (data: {
+    deploymentId: string;
+  }): Promise<{
+    applicationDeploymentRemovalRequestId: string;
+    applicationDeploymentRemovalRequestData: ApplicationDeploymentRemovalRequest;
+  }> {
+    const applicationDeploymentRemovalRequest = {
+      type: APP_DEPLOYMENT_REMOVAL_REQUEST_TYPE,
+      version: '1.0.0',
+      deployment: data.deploymentId
+    };
+
+    const result = await this.registry.setRecord(
+      {
+        privateKey: this.registryConfig.privateKey,
+        record: applicationDeploymentRemovalRequest,
+        bondId: this.registryConfig.bondId
+      },
+      '',
+      this.registryConfig.fee
+    );
+
+    log(`Application deployment removal request record published: ${result.data.id}`);
+    log('Application deployment removal request data:', applicationDeploymentRemovalRequest);
+
+    return {
+      applicationDeploymentRemovalRequestId: result.data.id,
+      applicationDeploymentRemovalRequestData: applicationDeploymentRemovalRequest
+    };
+  }
+
   getCrn (appName: string): string {
     assert(this.registryConfig.authority, "Authority doesn't exist");
     return `crn://${this.registryConfig.authority}/applications/${appName}`;

packages/backend/src/resolvers.ts

@@ -255,6 +255,20 @@ export const createResolvers = async (service: Service): Promise<any> => {
         }
       },
 
+      deleteDeployment: async (
+        _: any,
+        {
+          deploymentId
+        }: { deploymentId: string; }
+      ) => {
+        try {
+          return await service.deleteDeployment(deploymentId);
+        } catch (err) {
+          log(err);
+          return false;
+        }
+      },
+
       addDomain: async (
         _: any,
         { projectId, data }: { projectId: string; data: { name: string } }

packages/backend/src/schema.gql

@@ -19,6 +19,7 @@ enum DeploymentStatus {
   Building
   Ready
   Error
+  Deleting
 }
 
 enum DomainStatus {
@@ -209,6 +210,7 @@ type Mutation {
   deleteProject(projectId: String!): Boolean!
   deleteDomain(domainId: String!): Boolean!
   rollbackDeployment(projectId: String!, deploymentId: String!): Boolean!
+  deleteDeployment(deploymentId: String!): Boolean!
   addDomain(projectId: String!, data: AddDomainInput!): Boolean!
   updateDomain(domainId: String!, data: UpdateDomainInput!): Boolean!
   authenticateGitHub(code: String!): AuthResult!

packages/backend/src/service.ts

@@ -15,13 +15,16 @@ import { Permission, ProjectMember } from './entity/ProjectMember';
 import { User } from './entity/User';
 import { Registry } from './registry';
 import { GitHubConfig, RegistryConfig } from './config';
-import { AppDeploymentRecord, GitPushEventPayload, PackageJSON } from './types';
+import { AppDeploymentRecord, AppDeploymentRemovalRecord, GitPushEventPayload, PackageJSON } from './types';
 import { Role } from './entity/UserOrganization';
 
 const log = debug('snowball:service');
 
 const GITHUB_UNIQUE_WEBHOOK_ERROR = 'Hook already exists on this repository';
 
+// Define a constant for an hour in milliseconds
+const HOUR = 1000 * 60 * 60;
+
 interface Config {
   gitHubConfig: GitHubConfig;
   registryConfig: RegistryConfig;
@@ -49,6 +52,8 @@ export class Service {
   init (): void {
     // Start check for ApplicationDeploymentRecords asynchronously
     this.checkDeployRecordsAndUpdate();
+    // Start check for ApplicationDeploymentRemovalRecords asynchronously
+    this.checkDeploymentRemovalRecordsAndUpdate();
   }
 
   /**
@@ -60,14 +65,13 @@ export class Service {
 
   /**
    * Checks for ApplicationDeploymentRecord and update corresponding deployments
-   * Continues check in loop after a delay of DEPLOY_RECORD_CHECK_DELAY_MS
+   * Continues check in loop after a delay of registryConfig.fetchDeploymentRecordDelay
    */
   async checkDeployRecordsAndUpdate (): Promise<void> {
     // Fetch deployments in building state
     const deployments = await this.db.getDeployments({
       where: {
         status: DeploymentStatus.Building
-        // TODO: Fetch and check records for recent deployments
       }
     });
 
@@ -76,6 +80,28 @@ export class Service {
         `Found ${deployments.length} deployments in ${DeploymentStatus.Building} state`
       );
 
+      // Calculate a timestamp for one hour ago
+      const anHourAgo = Date.now() - HOUR;
+
+      // Filter out deployments started more than an hour ago and mark them as Error
+      const oldDeploymentsToUpdate = deployments.filter(
+        deployment => (Number(deployment.updatedAt) < anHourAgo)
+      )
+        .map((deployment) => {
+          return this.db.updateDeploymentById(deployment.id, {
+            status: DeploymentStatus.Error,
+            isCurrent: false
+          });
+        });
+
+      // If there are old deployments to update, log and perform the updates
+      if (oldDeploymentsToUpdate.length > 0) {
+        log(
+          `Cleaning up ${oldDeploymentsToUpdate.length} deployments stuck in ${DeploymentStatus.Building} state for over an hour`
+        );
+        await Promise.all(oldDeploymentsToUpdate);
+      }
+
       // Fetch ApplicationDeploymentRecord for deployments
       const records = await this.registry.getDeploymentRecords(deployments);
       log(`Found ${records.length} ApplicationDeploymentRecords`);
@@ -91,6 +117,38 @@ export class Service {
     }, this.config.registryConfig.fetchDeploymentRecordDelay);
   }
 
+  /**
+   * Checks for ApplicationDeploymentRemovalRecord and remove corresponding deployments
+   * Continues check in loop after a delay of registryConfig.fetchDeploymentRecordDelay
+   */
+  async checkDeploymentRemovalRecordsAndUpdate (): Promise<void> {
+    // Fetch deployments in deleting state
+    const deployments = await this.db.getDeployments({
+      where: {
+        status: DeploymentStatus.Deleting
+      }
+    });
+
+    if (deployments.length) {
+      log(
+        `Found ${deployments.length} deployments in ${DeploymentStatus.Deleting} state`
+      );
+
+      // Fetch ApplicationDeploymentRemovalRecords for deployments
+      const records = await this.registry.getDeploymentRemovalRecords(deployments);
+      log(`Found ${records.length} ApplicationDeploymentRemovalRecords`);
+
+      // Update deployments for which ApplicationDeploymentRemovalRecords were returned
+      if (records.length) {
+        await this.deleteDeploymentsWithRecordData(records, deployments);
+      }
+    }
+
+    this.deployRecordCheckTimeout = setTimeout(() => {
+      this.checkDeploymentRemovalRecordsAndUpdate();
+    }, this.config.registryConfig.fetchDeploymentRecordDelay);
+  }
+
   /**
    * Update deployments with ApplicationDeploymentRecord data
    */
@@ -153,6 +211,45 @@ export class Service {
     await Promise.all(deploymentUpdatePromises);
   }
 
+  /**
+   * Delete deployments with ApplicationDeploymentRemovalRecord data
+   */
+  async deleteDeploymentsWithRecordData (
+    records: AppDeploymentRemovalRecord[],
+    deployments: Deployment[],
+  ): Promise<void> {
+    const removedApplicationDeploymentRecordIds = records.map(record => record.attributes.deployment);
+    
+    // Get removed deployments for ApplicationDeploymentRecords
+    const removedDeployments = deployments.filter(deployment => removedApplicationDeploymentRecordIds.includes(deployment.applicationDeploymentRecordId!))
+
+    const recordToDeploymentsMap = removedDeployments.reduce(
+      (acc: { [key: string]: Deployment }, deployment) => {
+        acc[deployment.applicationDeploymentRecordId!] = deployment;
+        return acc;
+      },
+      {}
+    );
+
+    // Update deployment data for ApplicationDeploymentRecords and delete
+    const deploymentUpdatePromises = records.map(async (record) => {
+      const deployment = recordToDeploymentsMap[record.attributes.deployment];
+
+      await this.db.updateDeploymentById(deployment.id, {
+        applicationDeploymentRemovalRecordId: record.id,
+        applicationDeploymentRemovalRecordData: record.attributes,
+      });
+
+      log(
+        `Updated deployment ${deployment.id} with ApplicationDeploymentRemovalRecord ${record.id}`
+      );
+
+      await this.db.deleteDeploymentById(deployment.id)
+    });
+
+    await Promise.all(deploymentUpdatePromises);
+  }
+
   async getUser (userId: string): Promise<User | null> {
     return this.db.getUser({
       where: {
@@ -479,17 +576,10 @@ export class Service {
       return acc;
     }, {} as { [key: string]: string });
 
-    const { applicationDeploymentRequestId, applicationDeploymentRequestData } = await this.registry.createApplicationDeploymentRequest(
-      {
-        deployment: newDeployment,
-        appName: repo,
-        repository: repoUrl,
-        environmentVariables: environmentVariablesObj,
-        dns: `${newDeployment.project.name}-${newDeployment.id}`
-      });
-
     // To set project DNS
     if (data.environment === Environment.Production) {
+      // On deleting deployment later, project DNS deployment is also deleted
+      // So publish project DNS deployment first so that ApplicationDeploymentRecord for the same is available when deleting deployment later
       await this.registry.createApplicationDeploymentRequest(
         {
           deployment: newDeployment,
@@ -500,6 +590,15 @@ export class Service {
         });
     }
 
+    const { applicationDeploymentRequestId, applicationDeploymentRequestData } = await this.registry.createApplicationDeploymentRequest(
+      {
+        deployment: newDeployment,
+        appName: repo,
+        repository: repoUrl,
+        environmentVariables: environmentVariablesObj,
+        dns: `${newDeployment.project.name}-${newDeployment.id}`
+      });
+
     await this.db.updateDeploymentById(newDeployment.id, { applicationDeploymentRequestId, applicationDeploymentRequestData });
 
     return newDeployment;
@@ -717,6 +816,51 @@ export class Service {
     return newCurrentDeploymentUpdate && oldCurrentDeploymentUpdate;
   }
 
+  async deleteDeployment (deploymentId: string): Promise<boolean> {
+    const deployment = await this.db.getDeployment({
+      where: {
+        id: deploymentId
+      },
+      relations: {
+        project: true
+      }
+    });
+
+    if (deployment && deployment.applicationDeploymentRecordId) {
+      // If deployment is current, remove deployment for project subdomain as well
+      if (deployment.isCurrent) {
+        const currentDeploymentURL = `https://${deployment.project.subDomain}`;
+        
+        const deploymentRecords = await this.registry.getDeploymentRecordsByFilter({
+          application: deployment.applicationRecordId,
+          url: currentDeploymentURL
+        })
+
+        if (!deploymentRecords.length) {
+          log(`No ApplicationDeploymentRecord found for URL ${currentDeploymentURL} and ApplicationDeploymentRecord id ${deployment.applicationDeploymentRecordId}`);
+          return false;
+        }
+
+        await this.registry.createApplicationDeploymentRemovalRequest({ deploymentId: deploymentRecords[0].id });
+      }
+
+      const result = await this.registry.createApplicationDeploymentRemovalRequest({ deploymentId: deployment.applicationDeploymentRecordId });
+
+      await this.db.updateDeploymentById(
+        deployment.id,
+        {
+          status: DeploymentStatus.Deleting,
+          applicationDeploymentRemovalRequestId: result.applicationDeploymentRemovalRequestId,
+          applicationDeploymentRemovalRequestData: result.applicationDeploymentRemovalRequestData
+        }
+      );
+
+      return (result !== undefined || result !== null);
+    }
+
+    return false;
+  }
+
   async addDomain (
     projectId: string,
     data: { name: string }

packages/backend/src/types.ts

@@ -38,6 +38,13 @@ export interface AppDeploymentRecordAttributes {
   version: string;
 }
 
+export interface AppDeploymentRemovalRecordAttributes {
+  deployment: string;
+  request: string;
+  type: "ApplicationDeploymentRemovalRecord";
+  version: string;
+}
+
 interface RegistryRecord {
   id: string;
   names: string[] | null;
@@ -50,3 +57,7 @@ interface RegistryRecord {
 export interface AppDeploymentRecord extends RegistryRecord {
   attributes: AppDeploymentRecordAttributes;
 }
+
+export interface AppDeploymentRemovalRecord extends RegistryRecord {
+  attributes: AppDeploymentRemovalRecordAttributes;
+}

packages/backend/test/publish-deploy-records.ts

@@ -7,7 +7,7 @@ import { Registry } from '@snowballtools/laconic-sdk';
 import { Config } from '../src/config';
 import { DEFAULT_CONFIG_FILE_PATH } from '../src/constants';
 import { getConfig } from '../src/utils';
-import { Deployment, DeploymentStatus } from '../src/entity/Deployment';
+import { Deployment, DeploymentStatus, Environment } from '../src/entity/Deployment';
 
 const log = debug('snowball:publish-deploy-records');
 
@@ -40,7 +40,7 @@ async function main () {
   });
 
   for await (const deployment of deployments) {
-    const url = `${deployment.project.name}-${deployment.id}.${misc.projectDomain}`;
+    const url = `https://${deployment.project.name}-${deployment.id}.${misc.projectDomain}`;
 
     const applicationDeploymentRecord = {
       type: 'ApplicationDeploymentRecord',
@@ -71,6 +71,21 @@ async function main () {
       registryConfig.fee
     );
 
+    // Remove deployment for project subdomain if deployment is for production environment
+    if (deployment.environment === Environment.Production) {
+      applicationDeploymentRecord.url = `https://${deployment.project.subDomain}`
+
+      await registry.setRecord(
+        {
+          privateKey: registryConfig.privateKey,
+          record: applicationDeploymentRecord,
+          bondId: registryConfig.bondId
+        },
+        '',
+        registryConfig.fee
+      );
+    }
+
     log('Application deployment record data:', applicationDeploymentRecord);
     log(`Application deployment record published: ${result.data.id}`);
   }

packages/backend/test/publish-deployment-removal-records.ts

@@ -0,0 +1,67 @@
+import debug from 'debug';
+import { DataSource } from 'typeorm';
+import path from 'path';
+
+import { Registry } from '@cerc-io/laconic-sdk';
+
+import { Config } from '../src/config';
+import { DEFAULT_CONFIG_FILE_PATH } from '../src/constants';
+import { getConfig } from '../src/utils';
+import { Deployment, DeploymentStatus } from '../src/entity/Deployment';
+
+const log = debug('snowball:publish-deployment-removal-records');
+
+async function main () {
+  const { registryConfig, database, misc } = await getConfig<Config>(DEFAULT_CONFIG_FILE_PATH);
+
+  const registry = new Registry(
+    registryConfig.gqlEndpoint,
+    registryConfig.restEndpoint,
+    registryConfig.chainId
+  );
+
+  const dataSource = new DataSource({
+    type: 'better-sqlite3',
+    database: database.dbPath,
+    synchronize: true,
+    entities: [path.join(__dirname, '../src/entity/*')]
+  });
+
+  await dataSource.initialize();
+
+  const deploymentRepository = dataSource.getRepository(Deployment);
+  const deployments = await deploymentRepository.find({
+    relations: {
+      project: true
+    },
+    where: {
+      status: DeploymentStatus.Deleting
+    }
+  });
+
+  for await (const deployment of deployments) {
+    const applicationDeploymentRemovalRecord = {
+      type: "ApplicationDeploymentRemovalRecord",
+      version: "1.0.0",
+      deployment: deployment.applicationDeploymentRecordId,
+      request: deployment.applicationDeploymentRemovalRequestId,
+    }
+
+    const result = await registry.setRecord(
+      {
+        privateKey: registryConfig.privateKey,
+        record: applicationDeploymentRemovalRecord,
+        bondId: registryConfig.bondId
+      },
+      '',
+      registryConfig.fee
+    );
+
+    log('Application deployment removal record data:', applicationDeploymentRemovalRecord);
+    log(`Application deployment removal record published: ${result.data.id}`);
+  }
+}
+
+main().catch((err) => {
+  log(err);
+});