Add a script to get subscribed and onboarded laconicd accounts (#22)

Part of [laconicd testnet validator enrollment](cerc-io/testnet-laconicd-stack#19)

The script:
- Fetches onboarded participants using the configured laconicd GQL endpoint
- Reads a given CSV file with subscribers data
- Fetches all the onboarding txs using given laconicd RPC endpoint
- For each subscriber registered as a participant, maps the laconic pubkey and latest onboarding tx height
- Outputs the result in a CSV

Co-authored-by: Nabarun <nabarun@deepstacksoft.com>
Reviewed-on: cerc-io/testnet-laconicd-stack#22
Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com>
Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>

.gitignore

@@ -0,0 +1,2 @@
+*-deployment
+*-spec.yml

scripts/.env.example

@@ -0,0 +1,8 @@
+# Default: https://laconicd.laconic.com/api
+LACONICD_GQL_ENDPOINT=
+
+# Default: https://laconicd.laconic.com
+LACONICD_RPC_ENDPOINT=
+
+# Default: laconic_9000-1
+LACONICD_CHAIN_ID=

scripts/.gitignore

@@ -0,0 +1,3 @@
+node_modules
+dist
+.env

scripts/.npmrc

@@ -0,0 +1 @@
+@cerc-io:registry=https://git.vdb.to/api/packages/cerc-io/npm/

scripts/README.md

@@ -0,0 +1,36 @@
+# cli
+
+## Prerequisites
+
+- NodeJS >= `v18.17.x`
+
+## Instructions
+
+- Change to scripts dir:
+
+  ```bash
+  cd scripts
+  ```
+
+- Install dependencies and build:
+
+  ```bash
+  yarn && yarn build
+  ```
+
+- Create required env configuration:
+
+  ```bash
+  # Update the values as required
+  # By default, live laconicd testnet (laconicd.laconic.com) endpoint is configured
+  cp .env.example .env
+  ```
+
+- Map subscribers to onboarded participants:
+
+  ```bash
+  yarn map-subscribers-to-participants --subscribers-csv <subscribers-csv-file> --output <output-csv-file>
+
+  # Example:
+  # yarn map-subscribers-to-participants --subscribers-csv subscribers.csv --output result-$(date +"%Y-%m-%d%Y%m%d").csv
+  ```

scripts/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "testnet-laconicd-stack",
+  "version": "0.1.0",
+  "main": "index.js",
+  "repository": "git@git.vdb.to:cerc-io/testnet-laconicd-stack.git",
+  "license": "UNLICENSED",
+  "private": true,
+  "devDependencies": {
+    "@types/cli-progress": "^3.11.6",
+    "@types/node": "^22.2.0",
+    "@types/yargs": "^17.0.33",
+    "typescript": "^5.5.4"
+  },
+  "dependencies": {
+    "@cerc-io/registry-sdk": "^0.2.6",
+    "@cosmjs/stargate": "^0.32.4",
+    "csv-parse": "^5.5.6",
+    "csv-parser": "^3.0.0",
+    "csv-writer": "^1.6.0",
+    "dotenv": "^16.4.5",
+    "yargs": "^17.7.2"
+  },
+  "scripts": {
+    "build": "tsc",
+    "map-subscribers-to-participants": "node dist/map-subscribers-to-participants.js"
+  },
+  "packageManager": "yarn@1.22.19+sha1.4ba7fc5c6e704fce2066ecbfb0b0d8976fe62447"
+}

scripts/src/map-subscribers-to-participants.ts

@@ -0,0 +1,171 @@
+import * as fs from 'fs';
+import * as crypto from 'crypto';
+import * as path from 'path';
+import yargs from 'yargs';
+import { hideBin } from 'yargs/helpers';
+import { parse as csvParse } from 'csv-parse';
+import * as csvWriter from 'csv-writer';
+import dotenv from 'dotenv';
+
+import { StargateClient } from '@cosmjs/stargate';
+import { Registry } from '@cerc-io/registry-sdk';
+import { decodeTxRaw, decodePubkey } from '@cosmjs/proto-signing';
+
+dotenv.config();
+
+const LACONICD_GQL_ENDPOINT = process.env.LACONICD_GQL_ENDPOINT || 'https://laconicd.laconic.com/api';
+const LACONICD_RPC_ENDPOINT = process.env.LACONICD_RPC_ENDPOINT || 'https://laconicd.laconic.com';
+const LACONICD_CHAIN_ID = process.env.LACONICD_CHAIN_ID || 'laconic_9000-1';
+
+async function main(): Promise<void> {
+  const argv = _getArgv();
+
+  const registry = new Registry(LACONICD_GQL_ENDPOINT, LACONICD_RPC_ENDPOINT, LACONICD_CHAIN_ID);
+  const client = await StargateClient.connect(LACONICD_RPC_ENDPOINT);
+
+  console.time('time_taken_getParticipants');
+  const participants = await registry.getParticipants();
+  console.timeEnd('time_taken_getParticipants');
+
+  const subscribers = await readSubscribers(argv.subscribersCsv);
+  console.log('Read subscribers, count:', subscribers.length);
+
+  await processSubscribers(client, participants, subscribers, argv.output);
+}
+
+async function readSubscribers(subscribersCsvPath: string): Promise<any> {
+  const fileContent = fs.readFileSync(path.resolve(subscribersCsvPath), { encoding: 'utf-8' });
+  const headers = ['subscriber_id', 'email', 'status', 'premium?', 'created_at', 'api_subscription_id'];
+
+  return csvParse(fileContent, { delimiter: ',', columns: headers }).toArray();
+}
+
+function hashSubscriberId(subscriberId: string): string {
+  return '0x' + crypto.createHash('sha256').update(subscriberId).digest('hex');
+}
+
+async function processSubscribers(client: StargateClient, participants: any[], subscribers: any[], outputPath: string) {
+  // Map kyc_id to participant data
+  const kycMap: Record<string, any> = {};
+  participants.forEach((participant: any) => {
+    kycMap[participant.kycId] = participant;
+  });
+
+  const onboardingTxsHeightMap: Record<string, { txHeight: number, pubkey: string }> = {};
+  console.time('time_taken_searchTx');
+  const onboardingTxs = await client.searchTx(`message.action='/cerc.onboarding.v1.MsgOnboardParticipant'`);
+  console.timeEnd('time_taken_searchTx');
+
+  console.log('Fetched onboardingTxs, count:', onboardingTxs.length);
+
+  console.time('time_taken_decodingTxs');
+  onboardingTxs.forEach(onboardingTx => {
+    const rawPubkey = decodeTxRaw(onboardingTx.tx).authInfo.signerInfos[0].publicKey;
+    if (!rawPubkey) {
+      console.error('pubkey not found in tx', onboardingTx.hash);
+      return;
+    }
+
+    const pubkey = decodePubkey(rawPubkey).value;
+
+    // Determine sender
+    const onboardParticipantEvent = onboardingTx.events.find(event => event.type === 'onboard_participant');
+    if (!onboardParticipantEvent) {
+      console.error('onboard_participant event not found in tx', onboardingTx.hash);
+      return;
+    }
+
+    const sender = onboardParticipantEvent.attributes.find(attr => attr.key === 'signer')?.value;
+    if (!sender) {
+      console.error('sender not found in onboard_participant event for tx', onboardingTx.hash)
+      return;
+    }
+
+    // Update if already exists
+    let latesTxHeight = onboardingTx.height;
+    if (onboardingTxsHeightMap[sender]) {
+      latesTxHeight = latesTxHeight > onboardingTxsHeightMap[sender].txHeight ? latesTxHeight : onboardingTxsHeightMap[sender].txHeight;
+    }
+
+    onboardingTxsHeightMap[sender] = { txHeight: latesTxHeight, pubkey };
+  });
+  console.timeEnd('time_taken_decodingTxs');
+
+  const onboardedSubscribers: any[] = [];
+  for (let i = 0; i < subscribers.length; i++) {
+    const subscriber = subscribers[i];
+
+    const hashedSubscriberId = hashSubscriberId(subscriber['subscriber_id']);
+    const participant = kycMap[hashedSubscriberId];
+    if (!participant) {
+      continue;
+    }
+
+    const participantAddresss = participant['cosmosAddress'];
+
+    // Skip participant if an onboarding tx not found
+    if (!onboardingTxsHeightMap[participantAddresss]) {
+      continue;
+    }
+
+    const onboardedSubscriber = {
+      subscriber_id: subscriber['subscriber_id'],
+      email: subscriber['email'],
+      status: subscriber['status'],
+      'premium?': subscriber['premium?'],
+      created_at: subscriber['created_at'],
+      laconic_address: participantAddresss,
+      nitro_address: participant['nitroAddress'],
+      role: participant['role'],
+      hashed_subscriber_id: participant['kycId'],
+      laconic_pubkey: onboardingTxsHeightMap[participantAddresss].pubkey,
+      onboarding_height: onboardingTxsHeightMap[participantAddresss].txHeight
+    };
+
+    onboardedSubscribers.push(onboardedSubscriber);
+  }
+
+  const writer = csvWriter.createObjectCsvWriter({
+    path: path.resolve(outputPath),
+    header: [
+      { id: 'subscriber_id', title: 'subscriber_id' },
+      { id: 'email', title: 'email' },
+      { id: 'status', title: 'status' },
+      { id: 'premium?', title: 'premium?' },
+      { id: 'created_at', title: 'created_at' },
+      { id: 'laconic_address', title: 'laconic_address' },
+      { id: 'nitro_address', title: 'nitro_address' },
+      { id: 'role', title: 'role' },
+      { id: 'hashed_subscriber_id', title: 'hashed_subscriber_id' },
+      { id: 'laconic_pubkey', title: 'laconic_pubkey' },
+      { id: 'onboarding_height', title: 'onboarding_height' },
+    ],
+    alwaysQuote: true
+  });
+
+  await writer.writeRecords(onboardedSubscribers);
+
+  console.log(`Data has been written to ${path.resolve(outputPath)}`);
+}
+
+function _getArgv (): any {
+  return yargs(hideBin(process.argv))
+  .option('subscribersCsv', {
+    alias: 's',
+    type: 'string',
+    demandOption: true,
+    describe: 'Path to the subscribers CSV file',
+  })
+  .option('output', {
+    alias: 'o',
+    type: 'string',
+    demandOption: true,
+    describe: 'Path to the output CSV file',
+  })
+  .help()
+  .argv;
+}
+
+main().catch(err => {
+  console.log(err);
+});

scripts/tsconfig.json

@@ -0,0 +1,110 @@
+{
+  "compilerOptions": {
+    /* Visit https://aka.ms/tsconfig to read more about this file */
+
+    /* Projects */
+    // "incremental": true,                              /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
+    // "composite": true,                                /* Enable constraints that allow a TypeScript project to be used with project references. */
+    // "tsBuildInfoFile": "./.tsbuildinfo",              /* Specify the path to .tsbuildinfo incremental compilation file. */
+    // "disableSourceOfProjectReferenceRedirect": true,  /* Disable preferring source files instead of declaration files when referencing composite projects. */
+    // "disableSolutionSearching": true,                 /* Opt a project out of multi-project reference checking when editing. */
+    // "disableReferencedProjectLoad": true,             /* Reduce the number of projects loaded automatically by TypeScript. */
+
+    /* Language and Environment */
+    "target": "es2016",                                  /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
+    // "lib": [],                                        /* Specify a set of bundled library declaration files that describe the target runtime environment. */
+    // "jsx": "preserve",                                /* Specify what JSX code is generated. */
+    // "experimentalDecorators": true,                   /* Enable experimental support for legacy experimental decorators. */
+    // "emitDecoratorMetadata": true,                    /* Emit design-type metadata for decorated declarations in source files. */
+    // "jsxFactory": "",                                 /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */
+    // "jsxFragmentFactory": "",                         /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
+    // "jsxImportSource": "",                            /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */
+    // "reactNamespace": "",                             /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */
+    // "noLib": true,                                    /* Disable including any library files, including the default lib.d.ts. */
+    // "useDefineForClassFields": true,                  /* Emit ECMAScript-standard-compliant class fields. */
+    // "moduleDetection": "auto",                        /* Control what method is used to detect module-format JS files. */
+
+    /* Modules */
+    "module": "commonjs",                                /* Specify what module code is generated. */
+    // "rootDir": "./",                                  /* Specify the root folder within your source files. */
+    // "moduleResolution": "node10",                     /* Specify how TypeScript looks up a file from a given module specifier. */
+    // "baseUrl": "./",                                  /* Specify the base directory to resolve non-relative module names. */
+    // "paths": {},                                      /* Specify a set of entries that re-map imports to additional lookup locations. */
+    // "rootDirs": [],                                   /* Allow multiple folders to be treated as one when resolving modules. */
+    // "typeRoots": [],                                  /* Specify multiple folders that act like './node_modules/@types'. */
+    // "types": [],                                      /* Specify type package names to be included without being referenced in a source file. */
+    // "allowUmdGlobalAccess": true,                     /* Allow accessing UMD globals from modules. */
+    // "moduleSuffixes": [],                             /* List of file name suffixes to search when resolving a module. */
+    // "allowImportingTsExtensions": true,               /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */
+    // "resolvePackageJsonExports": true,                /* Use the package.json 'exports' field when resolving package imports. */
+    // "resolvePackageJsonImports": true,                /* Use the package.json 'imports' field when resolving imports. */
+    // "customConditions": [],                           /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */
+    "resolveJsonModule": true,                        /* Enable importing .json files. */
+    // "allowArbitraryExtensions": true,                 /* Enable importing files with any extension, provided a declaration file is present. */
+    // "noResolve": true,                                /* Disallow 'import's, 'require's or '<reference>'s from expanding the number of files TypeScript should add to a project. */
+
+    /* JavaScript Support */
+    // "allowJs": true,                                  /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */
+    // "checkJs": true,                                  /* Enable error reporting in type-checked JavaScript files. */
+    // "maxNodeModuleJsDepth": 1,                        /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */
+
+    /* Emit */
+    "declaration": true,                              /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
+    // "declarationMap": true,                           /* Create sourcemaps for d.ts files. */
+    // "emitDeclarationOnly": true,                      /* Only output d.ts files and not JavaScript files. */
+    "sourceMap": true,                                /* Create source map files for emitted JavaScript files. */
+    // "inlineSourceMap": true,                          /* Include sourcemap files inside the emitted JavaScript. */
+    // "outFile": "./",                                  /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
+    "outDir": "dist",                                   /* Specify an output folder for all emitted files. */
+    // "removeComments": true,                           /* Disable emitting comments. */
+    // "noEmit": true,                                   /* Disable emitting files from a compilation. */
+    // "importHelpers": true,                            /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
+    // "downlevelIteration": true,                       /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
+    // "sourceRoot": "",                                 /* Specify the root path for debuggers to find the reference source code. */
+    // "mapRoot": "",                                    /* Specify the location where debugger should locate map files instead of generated locations. */
+    // "inlineSources": true,                            /* Include source code in the sourcemaps inside the emitted JavaScript. */
+    // "emitBOM": true,                                  /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
+    // "newLine": "crlf",                                /* Set the newline character for emitting files. */
+    // "stripInternal": true,                            /* Disable emitting declarations that have '@internal' in their JSDoc comments. */
+    // "noEmitHelpers": true,                            /* Disable generating custom helper functions like '__extends' in compiled output. */
+    // "noEmitOnError": true,                            /* Disable emitting files if any type checking errors are reported. */
+    // "preserveConstEnums": true,                       /* Disable erasing 'const enum' declarations in generated code. */
+    // "declarationDir": "./",                           /* Specify the output directory for generated declaration files. */
+
+    /* Interop Constraints */
+    // "isolatedModules": true,                          /* Ensure that each file can be safely transpiled without relying on other imports. */
+    // "verbatimModuleSyntax": true,                     /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */
+    // "isolatedDeclarations": true,                     /* Require sufficient annotation on exports so other tools can trivially generate declaration files. */
+    // "allowSyntheticDefaultImports": true,             /* Allow 'import x from y' when a module doesn't have a default export. */
+    "esModuleInterop": true,                             /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */
+    // "preserveSymlinks": true,                         /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
+    "forceConsistentCasingInFileNames": true,            /* Ensure that casing is correct in imports. */
+
+    /* Type Checking */
+    "strict": true,                                      /* Enable all strict type-checking options. */
+    // "noImplicitAny": true,                            /* Enable error reporting for expressions and declarations with an implied 'any' type. */
+    // "strictNullChecks": true,                         /* When type checking, take into account 'null' and 'undefined'. */
+    // "strictFunctionTypes": true,                      /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
+    // "strictBindCallApply": true,                      /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */
+    // "strictPropertyInitialization": true,             /* Check for class properties that are declared but not set in the constructor. */
+    // "noImplicitThis": true,                           /* Enable error reporting when 'this' is given the type 'any'. */
+    // "useUnknownInCatchVariables": true,               /* Default catch clause variables as 'unknown' instead of 'any'. */
+    // "alwaysStrict": true,                             /* Ensure 'use strict' is always emitted. */
+    // "noUnusedLocals": true,                           /* Enable error reporting when local variables aren't read. */
+    // "noUnusedParameters": true,                       /* Raise an error when a function parameter isn't read. */
+    // "exactOptionalPropertyTypes": true,               /* Interpret optional property types as written, rather than adding 'undefined'. */
+    // "noImplicitReturns": true,                        /* Enable error reporting for codepaths that do not explicitly return in a function. */
+    // "noFallthroughCasesInSwitch": true,               /* Enable error reporting for fallthrough cases in switch statements. */
+    // "noUncheckedIndexedAccess": true,                 /* Add 'undefined' to a type when accessed using an index. */
+    // "noImplicitOverride": true,                       /* Ensure overriding members in derived classes are marked with an override modifier. */
+    // "noPropertyAccessFromIndexSignature": true,       /* Enforces using indexed accessors for keys declared using an indexed type. */
+    // "allowUnusedLabels": true,                        /* Disable error reporting for unused labels. */
+    // "allowUnreachableCode": true,                     /* Disable error reporting for unreachable code. */
+
+    /* Completeness */
+    // "skipDefaultLibCheck": true,                      /* Skip type checking .d.ts files that are included with TypeScript. */
+    "skipLibCheck": true                                 /* Skip type checking all .d.ts files. */
+  },
+  "include": ["src"],
+  "exclude": ["dist"],
+}