Solidity data mapper/parser (#12)

* Initial setup with hardhat and typescript.

* Add test for integer type.

* Add test for unsigned integer type.

* Add test for boolean type.

* Add test for address type.

* Add test for string type.

* Setup building library with typescript.

* Remove hardhat dependency from getStorageValue library function.

* Move contracts to test and remove deploy script.

* Add readme for running tests.

Co-authored-by: nikugogoi <95nikass@gmail.com>

packages/solidity-mapper/.gitignore

@@ -0,0 +1,8 @@
+node_modules
+
+#Hardhat files
+cache
+artifacts
+
+#yarn
+yarn-error.log

packages/solidity-mapper/README.md

@@ -0,0 +1,38 @@
+# solidity-mapper
+
+Get value of state variable from storage for a solidity contract.
+
+## Pre-requisites
+
+* NodeJS and NPM
+
+  https://nodejs.org/en/ or use https://github.com/nvm-sh/nvm
+
+## Instructions
+
+Run the tests using the following command
+```bash
+$ yarn test
+```
+
+## Different Types
+
+* Booleans
+* Integers
+* Fixed Point Numbers
+* Address
+* Contract Types
+* Fixed-size byte arrays
+* Enums
+* Function Types
+* Arrays
+* Structs
+* Mapping Types
+
+## Observations
+
+* The storage layouts are formed according to the rules in https://docs.soliditylang.org/en/latest/internals/layout_in_storage.html#layout-of-state-variables-in-storage
+
+* Structs can occupy multiple slots depending on the size required by its members.
+
+* Fixed arrays can occupy multiple slots according to the size of the array and the type of array.

packages/solidity-mapper/hardhat.config.ts

@@ -0,0 +1,41 @@
+import { task, HardhatUserConfig } from "hardhat/config";
+import "@nomiclabs/hardhat-waffle";
+
+// This is a sample Hardhat task. To learn how to create your own go to
+// https://hardhat.org/guides/create-task.html
+task("accounts", "Prints the list of accounts", async (args, hre) => {
+  const accounts = await hre.ethers.getSigners();
+
+  for (const account of accounts) {
+    console.log(account.address);
+  }
+});
+
+// You need to export an object to set up your config
+// Go to https://hardhat.org/config/ to learn more
+
+const config: HardhatUserConfig = {
+  solidity: {
+    version: "0.7.3",
+    settings: {
+      outputSelection: {
+        "*": {
+          "*": [
+            "abi", "storageLayout",
+            "metadata", "evm.bytecode", // Enable the metadata and bytecode outputs of every single contract.
+            "evm.bytecode.sourceMap" // Enable the source map output of every single contract.
+          ],
+          "": [
+            "ast" // Enable the AST output of every single file.
+          ]
+        }
+      },
+    }
+  },
+  paths: {
+    sources: './test/contracts',
+    tests: './src'
+  }
+};
+
+export default config;

packages/solidity-mapper/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "@vulcanize/solidity-mapper",
+  "version": "0.1.0",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "license": "UNLICENSED",
+  "devDependencies": {
+    "@nomiclabs/hardhat-ethers": "^2.0.2",
+    "@nomiclabs/hardhat-waffle": "^2.0.1",
+    "@types/chai": "^4.2.18",
+    "@types/mocha": "^8.2.2",
+    "@types/node": "^15.6.1",
+    "chai": "^4.3.4",
+    "ethereum-waffle": "^3.3.0",
+    "ethers": "^5.2.0",
+    "hardhat": "^2.3.0",
+    "ts-node": "^10.0.0",
+    "typescript": "^4.3.2"
+  },
+  "scripts": {
+    "build": "tsc",
+    "test": "hardhat test"
+  }
+}

packages/solidity-mapper/src/index.ts

@@ -0,0 +1 @@
+export { getStorageValue, StorageLayout, GetStorageAt } from './storage';

packages/solidity-mapper/src/storage.test.ts

@@ -0,0 +1,89 @@
+import { Contract } from "@ethersproject/contracts";
+import { expect } from "chai";
+import hre from "hardhat";
+import "@nomiclabs/hardhat-ethers";
+
+import { getStorageValue, StorageLayout } from "./storage";
+import { getStorageLayout, getStorageAt } from "../test/utils";
+
+describe("Storage", function() {
+  it("get value for integer type", async function() {
+    const Integers = await hre.ethers.getContractFactory("TestIntegers");
+    const integers = await Integers.deploy();
+    await integers.deployed();
+    const storageLayout = await getStorageLayout("TestIntegers");
+
+    // if (storageLayout)
+    let value = 12;
+    await integers.setInt1(value);
+    let storageValue = await getStorageValue(integers.address, storageLayout, getStorageAt, "int1");
+    expect(storageValue).to.equal(value);
+  });
+
+  it("get value for unsigned integer type", async function() {
+    const UnsignedIntegers = await hre.ethers.getContractFactory("TestUnsignedIntegers");
+    const unsignedIntegers = await UnsignedIntegers.deploy();
+    await unsignedIntegers.deployed();
+    const storageLayout = await getStorageLayout("TestUnsignedIntegers");
+
+    const value = 123;
+    await unsignedIntegers.setUint1(value);
+    const storageValue = await getStorageValue(unsignedIntegers.address, storageLayout, getStorageAt, "uint1");
+    expect(storageValue).to.equal(value);
+  });
+
+  it("get value for boolean type", async function() {
+    const Booleans = await hre.ethers.getContractFactory("TestBooleans");
+    const booleans = await Booleans.deploy();
+    await booleans.deployed();
+    const storageLayout = await getStorageLayout("TestBooleans");
+
+    let value = true
+    await booleans.setBool1(value);
+    let storageValue = await getStorageValue(booleans.address, storageLayout, getStorageAt, "bool1");
+    expect(storageValue).to.equal(value)
+
+    value = false
+    await booleans.setBool2(value);
+    storageValue = await getStorageValue(booleans.address, storageLayout, getStorageAt, "bool2")
+    expect(storageValue).to.equal(value)
+  });
+
+  it("get value for address type", async function() {
+    const Address = await hre.ethers.getContractFactory("TestAddress");
+    const address = await Address.deploy();
+    await address.deployed();
+    const storageLayout = await getStorageLayout("TestAddress");
+
+    const [signer] = await hre.ethers.getSigners();
+    await address.setAddress1(signer.address);
+    const storageValue = await getStorageValue(address.address, storageLayout, getStorageAt, "address1");
+    expect(storageValue).to.be.a('string');
+    expect(String(storageValue).toLowerCase()).to.equal(signer.address.toLowerCase());
+  });
+
+  describe("string type", function () {
+    let strings: Contract, storageLayout: StorageLayout;
+
+    before(async () => {
+      const Strings = await hre.ethers.getContractFactory("TestStrings");
+      strings = await Strings.deploy();
+      await strings.deployed();
+      storageLayout = await getStorageLayout("TestStrings");
+    })
+
+    it("get value for string length less than 32 bytes", async function() {
+      const value = 'Hello world.'
+      await strings.setString1(value);
+      const storageValue = await getStorageValue(strings.address, storageLayout, getStorageAt, "string1");
+      expect(storageValue).to.equal(value);
+    });
+
+    it("get value for string length more than 32 bytes", async function() {
+      const value = 'This sentence is more than 32 bytes long.'
+      await strings.setString2(value);
+      const storageValue = await getStorageValue(strings.address, storageLayout, getStorageAt, "string2");
+      expect(storageValue).to.equal(value);
+    });
+  })
+});

packages/solidity-mapper/src/storage.ts

@@ -0,0 +1,136 @@
+import { utils, BigNumber } from 'ethers';
+
+export interface StorageLayout {
+  storage: [{
+    slot: string;
+    offset: number;
+    type: string;
+    label: string;
+  }];
+  types: {
+    [type: string]: {
+      encoding: string;
+      numberOfBytes: string;
+      label: string;
+    }
+  };
+}
+
+export type GetStorageAt = (address: string, position: string) => Promise<string>
+
+/**
+ * Function to get the value from storage for a contract variable.
+ * @param address
+ * @param storageLayout
+ * @param getStorageAt
+ * @param variableName
+ */
+export const getStorageValue = async (address: string, storageLayout: StorageLayout, getStorageAt: GetStorageAt, variableName: string): Promise<number | string | boolean | undefined> => {
+  const { storage, types } = storageLayout;
+  const targetState = storage.find((state) => state.label === variableName)
+
+  // Return if state variable could not be found in storage layout.
+  if (!targetState) {
+    return;
+  }
+
+  const { slot, offset, type } = targetState;
+  const { encoding, numberOfBytes, label } = types[type]
+
+  // Get value according to encoding i.e. how the data is encoded in storage.
+  // https://docs.soliditylang.org/en/v0.8.4/internals/layout_in_storage.html#json-output
+  switch (encoding) {
+    // https://docs.soliditylang.org/en/v0.8.4/internals/layout_in_storage.html#layout-of-state-variables-in-storage
+    case 'inplace': {
+      const valueArray = await getInplaceArray(address, slot, offset, numberOfBytes, getStorageAt);
+
+      // Parse value for address type.
+      if (['address', 'address payable'].some(type => type === label)) {
+        return utils.hexlify(valueArray);
+      }
+
+      // Parse value for boolean type.
+      if (label === 'bool') {
+        return !BigNumber.from(valueArray).isZero();
+      }
+
+      // Parse value for uint/int type.
+      return BigNumber.from(valueArray).toNumber();
+    }
+
+    // https://docs.soliditylang.org/en/v0.8.4/internals/layout_in_storage.html#bytes-and-string
+    case 'bytes': {
+      const valueArray = await getBytesArray(address, slot, getStorageAt);
+
+      return utils.toUtf8String(valueArray)
+    }
+
+    default:
+      break;
+  }
+}
+
+/**
+ * Function to get array value for inplace encoding.
+ * @param address
+ * @param slot
+ * @param offset
+ * @param numberOfBytes
+ * @param getStorageAt
+ */
+const getInplaceArray = async (address: string, slot: string, offset: number, numberOfBytes: string, getStorageAt: GetStorageAt) => {
+  const value = await getStorageAt(address, BigNumber.from(slot).toHexString());
+  const uintArray = utils.arrayify(value);
+
+  // Get value according to offset.
+  const start = uintArray.length - (offset + Number(numberOfBytes));
+  const end = uintArray.length - offset;
+  const offsetArray = uintArray.slice(start, end)
+
+  return offsetArray;
+}
+
+/**
+ * Function to get array value for bytes encoding.
+ * @param address
+ * @param slot
+ * @param getStorageAt
+ */
+const getBytesArray = async (address: string, slot: string, getStorageAt: GetStorageAt) => {
+  let value = await getStorageAt(address, BigNumber.from(slot).toHexString());
+  const uintArray = utils.arrayify(value);
+  let length = 0;
+
+  // Get length of bytes stored.
+  if (BigNumber.from(uintArray[0]).isZero()) {
+    // If first byte is not set, get length directly from the zero padded byte array.
+    const slotValue = BigNumber.from(value);
+    length = slotValue.sub(1).div(2).toNumber();
+  } else {
+    // If first byte is set the length is lesser than 32 bytes.
+    // Length of the value can be computed from the last byte.
+    length = BigNumber.from(uintArray[uintArray.length - 1]).div(2).toNumber();
+  }
+
+  // Get value from the byte array directly if length is less than 32.
+  if (length < 32) {
+    return uintArray.slice(0, length);
+  }
+
+  // Array to hold multiple bytes32 data.
+  const values = [];
+
+  // Compute zero padded hexstring to calculate hashed position of storage.
+  // https://github.com/ethers-io/ethers.js/issues/1079#issuecomment-703056242
+  const slotHex = utils.hexZeroPad(BigNumber.from(slot).toHexString(), 32);
+  const position = utils.keccak256(slotHex);
+
+  // Get value from consecutive storage slots for longer data.
+  for(let i = 0; i < length / 32; i++) {
+    const value = await getStorageAt(address, BigNumber.from(position).add(i).toHexString());
+    values.push(value);
+  }
+
+  // Slice trailing bytes according to length of value.
+  return utils.concat(values).slice(0, length);
+}

packages/solidity-mapper/test/contracts/TestAddress.sol

@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: MIT
+pragma solidity ^0.7.0;
+
+contract TestAddress {
+    address address1;
+
+    address payable address2;
+
+    // Set variable address1.
+    function setAddress1(address value) external {
+        address1 = value;
+    }
+
+    // Set variable address2.
+    function setAddress2(address payable value) external {
+        address2 = value;
+    }
+}

packages/solidity-mapper/test/contracts/TestBooleans.sol

@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: MIT
+pragma solidity ^0.7.0;
+
+contract TestBooleans {
+    // Variables are packed together in a slot as they occupy less than 32 bytes together.
+    bool bool1;
+    bool bool2;
+
+    // Set variable bool1.
+    function setBool1(bool value) external {
+        bool1 = value;
+    }
+
+    // Set variable bool2.
+    function setBool2(bool value) external {
+        bool2 = value;
+    }
+}

packages/solidity-mapper/test/contracts/TestIntegers.sol

@@ -0,0 +1,34 @@
+// SPDX-License-Identifier: MIT
+pragma solidity ^0.7.0;
+
+contract TestIntegers {
+    // Following variables are packed together in a single slot since the combined size is less than 32 bytes.
+    int8 int1;
+    int16 int2;
+
+    // Variable is stored in the next slot as it needs 32 bytes of storage.
+    int256 int3;
+
+    // Variable is stored in the next slot as there is not enough space for it in the previous slot.
+    int32 int4;
+
+    // Set variable int1.
+    function setInt1(int8 value) external {
+        int1 = value;
+    }
+
+    // Set variable int2.
+    function setInt2(int16 value) external {
+        int2 = value;
+    }
+
+    // Set variable int3.
+    function setInt3(int256 value) external {
+        int3 = value;
+    }
+
+    // Set variable int4.
+    function setInt4(int32 value) external {
+        int4 = value;
+    }
+}

packages/solidity-mapper/test/contracts/TestStrings.sol

@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: MIT
+pragma solidity ^0.7.0;
+
+contract TestStrings {
+    string string1;
+
+    string string2;
+
+    // Set variable string1.
+    function setString1(string memory value) external {
+        string1 = value;
+    }
+
+    // Set variable string2.
+    function setString2(string memory value) external {
+        string2 = value;
+    }
+}

packages/solidity-mapper/test/contracts/TestUnsignedIntegers.sol

@@ -0,0 +1,34 @@
+// SPDX-License-Identifier: MIT
+pragma solidity ^0.7.0;
+
+contract TestUnsignedIntegers {
+    // Following variables are packed together in a single slot since the combined size is less than 32 bytes.
+    uint8 uint1;
+    uint16 uint2;
+
+    // Variable is stored in the next slot as it needs 32 bytes of storage.
+    uint256 uint3;
+
+    // Variable is stored in the next slot as there is not enough space for it in the previous slot.
+    uint32 uint4;
+
+    // Set variable uint1.
+    function setUint1(uint8 value) external {
+        uint1 = value;
+    }
+
+    // Set variable uint2.
+    function setUint2(uint16 value) external {
+        uint2 = value;
+    }
+
+    // Set variable uint3.
+    function setUint3(uint256 value) external {
+        uint3 = value;
+    }
+
+    // Set variable uint4.
+    function setUint4(uint32 value) external {
+        uint4 = value;
+    }
+}

packages/solidity-mapper/test/utils.ts

@@ -0,0 +1,57 @@
+import { artifacts, ethers } from 'hardhat'
+import { CompilerOutput, CompilerOutputBytecode } from 'hardhat/types';
+
+import { StorageLayout, GetStorageAt } from '../src';
+
+// storageLayout doesnt exist in type CompilerOutput doesnt.
+// Extending CompilerOutput type to include storageLayout property.
+interface StorageCompilerOutput extends CompilerOutput {
+  contracts: {
+    [sourceName: string]: {
+      [contractName: string]: {
+        abi: any;
+        evm: {
+          bytecode: CompilerOutputBytecode;
+          deployedBytecode: CompilerOutputBytecode;
+          methodIdentifiers: {
+            [methodSignature: string]: string;
+          };
+        };
+        storageLayout?: StorageLayout;
+      }
+    };
+  };
+}
+
+/**
+ * Get storage layout of specified contract.
+ * @param contractName
+ */
+export const getStorageLayout = async (contractName: string) => {
+  const artifact = await artifacts.readArtifact(contractName);
+  const buildInfo = await artifacts.getBuildInfo(`${artifact.sourceName}:${artifact.contractName}`)
+
+  if (!buildInfo) {
+    throw new Error('storageLayout not present in compiler output.');
+  }
+
+  const output: StorageCompilerOutput = buildInfo.output
+  const { storageLayout } = output.contracts[artifact.sourceName][artifact.contractName];
+
+  if (!storageLayout) {
+    throw new Error(`Contract hasn't been compiled.`);
+  }
+
+  return storageLayout;
+}
+
+/**
+ * Get storage value in hardhat environment using ethers.
+ * @param address
+ * @param position
+ */
+export const getStorageAt: GetStorageAt = async (address, position) => {
+  const value = await ethers.provider.getStorageAt(address, position);
+
+  return value;
+}

packages/solidity-mapper/tsconfig.json

@@ -0,0 +1,74 @@
+{
+  "compilerOptions": {
+    /* Visit https://aka.ms/tsconfig.json to read more about this file */
+
+    /* Basic Options */
+    // "incremental": true,                         /* Enable incremental compilation */
+    "target": "es5",                                /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', 'ES2021', or 'ESNEXT'. */
+    "module": "commonjs",                           /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */
+    // "lib": [],                                   /* Specify library files to be included in the compilation. */
+    // "allowJs": true,                             /* Allow javascript files to be compiled. */
+    // "checkJs": true,                             /* Report errors in .js files. */
+    // "jsx": "preserve",                           /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */
+    "declaration": true,                         /* Generates corresponding '.d.ts' file. */
+    // "declarationMap": true,                      /* Generates a sourcemap for each corresponding '.d.ts' file. */
+    // "sourceMap": true,                           /* Generates corresponding '.map' file. */
+    // "outFile": "./",                             /* Concatenate and emit output to single file. */
+    "outDir": "./dist",                              /* Redirect output structure to the directory. */
+    // "rootDir": "./",                             /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
+    // "composite": true,                           /* Enable project compilation */
+    // "tsBuildInfoFile": "./",                     /* Specify file to store incremental compilation information */
+    // "removeComments": true,                      /* Do not emit comments to output. */
+    // "noEmit": true,                              /* Do not emit outputs. */
+    // "importHelpers": true,                       /* Import emit helpers from 'tslib'. */
+    // "downlevelIteration": true,                  /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
+    // "isolatedModules": true,                     /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
+
+    /* Strict Type-Checking Options */
+    "strict": true,                                 /* Enable all strict type-checking options. */
+    // "noImplicitAny": true,                       /* Raise error on expressions and declarations with an implied 'any' type. */
+    // "strictNullChecks": true,                    /* Enable strict null checks. */
+    // "strictFunctionTypes": true,                 /* Enable strict checking of function types. */
+    // "strictBindCallApply": true,                 /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
+    // "strictPropertyInitialization": true,        /* Enable strict checking of property initialization in classes. */
+    // "noImplicitThis": true,                      /* Raise error on 'this' expressions with an implied 'any' type. */
+    // "alwaysStrict": true,                        /* Parse in strict mode and emit "use strict" for each source file. */
+
+    /* Additional Checks */
+    // "noUnusedLocals": true,                      /* Report errors on unused locals. */
+    // "noUnusedParameters": true,                  /* Report errors on unused parameters. */
+    // "noImplicitReturns": true,                   /* Report error when not all code paths in function return a value. */
+    // "noFallthroughCasesInSwitch": true,          /* Report errors for fallthrough cases in switch statement. */
+    // "noUncheckedIndexedAccess": true,            /* Include 'undefined' in index signature results */
+    // "noImplicitOverride": true,                  /* Ensure overriding members in derived classes are marked with an 'override' modifier. */
+    // "noPropertyAccessFromIndexSignature": true,  /* Require undeclared properties from index signatures to use element accesses. */
+
+    /* Module Resolution Options */
+    "moduleResolution": "node",                  /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
+    // "baseUrl": "./",                             /* Base directory to resolve non-absolute module names. */
+    // "paths": {},                                 /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
+    // "rootDirs": [],                              /* List of root folders whose combined content represents the structure of the project at runtime. */
+    // "typeRoots": [],                             /* List of folders to include type definitions from. */
+    // "types": [],                                 /* Type declaration files to be included in compilation. */
+    // "allowSyntheticDefaultImports": true,        /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
+    "esModuleInterop": true,                        /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */
+    // "preserveSymlinks": true,                    /* Do not resolve the real path of symlinks. */
+    // "allowUmdGlobalAccess": true,                /* Allow accessing UMD globals from modules. */
+
+    /* Source Map Options */
+    // "sourceRoot": "",                            /* Specify the location where debugger should locate TypeScript files instead of source locations. */
+    // "mapRoot": "",                               /* Specify the location where debugger should locate map files instead of generated locations. */
+    // "inlineSourceMap": true,                     /* Emit a single file with source maps instead of having a separate file. */
+    // "inlineSources": true,                       /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
+
+    /* Experimental Options */
+    // "experimentalDecorators": true,              /* Enables experimental support for ES7 decorators. */
+    // "emitDecoratorMetadata": true,               /* Enables experimental support for emitting type metadata for decorators. */
+
+    /* Advanced Options */
+    "skipLibCheck": true,                           /* Skip type checking of declaration files. */
+    "forceConsistentCasingInFileNames": true        /* Disallow inconsistently-cased references to the same file. */
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules"],
+}