tests: refactor solidity test cases (#249)

* Refactor

* add script to run all tests

* Spawn ethermintd in node script

* Update README

* kill process when test finished

* add new test case

* add yarn.lock inside tests to be tracked

tests/solidity/.gitignore

@@ -3,3 +3,4 @@ node_modules/
 
 # ignore package-lock files (only use yarn.lock)
 package-lock.json
+!yarn.lock

tests/solidity/README.md

@@ -14,31 +14,31 @@ Increasingly difficult tests are provided:
 
 **Prerequisite**: install the individual solidity packages. They're set up as individual reops in a yarn monorepo workspace. Install them all via `yarn install`.
 
-To run the tests, start three terminals (or two, if you run `ethermintd` with `&`).
+To run the tests, you can use the `test-helper.js` utility to test all suites under `ganache` or `ethermint` network. The `test-helper.js` will help you spawn an `ethermintd` process before running the tests.
 
-In the first, run `ethermintd`:
+You can simply run `yarn test --network ethermint` to run all tests with ethermint network, or you can run `yarn test --network ganache` to use ganache shipped with truffle. In most cases, there two networks should produce identical test results. 
+
+If you only want to run a few test cases, append the name of tests following by the command line. For example, use `yarn test --network ethermint basic` to run the `basic` test under `ethermint` network.
+
+If you need to take more control, you can also run `ethermintd` using:
 
 ```sh
 ./init-test-node.sh
 ```
 
-In the second, run `ethermintd` as mentioned in the script's output:
-
-```sh
-ethermintd rest-server --laddr "tcp://localhost:8545" --unlock-key localkey,user1,user2 --chain-id "ethermint-1337" --trace --wsport 8546
-```
-
 You will now have three ethereum accounts unlocked in the test node:
 
 - `0x3b7252d007059ffc82d16d022da3cbf9992d2f70` (Validator)
 - `0xddd64b4712f7c8f1ace3c145c950339eddaf221d` (User 1)
 - `0x0f54f47bf9b8e317b214ccd6a7c3e38b893cd7f0` (user 2)
 
-From here, in your other available terminal, go into any of the tests and run `yarn test-ethermint`. You should see `ethermintd` accepting transactions and producing blocks. You should be able to query for any transaction via:
+
+Keep the terminal window open, go into any of the tests and run `yarn test-ethermint`. You should see `ethermintd` accepting transactions and producing blocks. You should be able to query for any transaction via:
 
 - `ethermintd query tx <cosmos-sdk tx>`
 - `curl localhost:8545 -H "Content-Type:application/json" -X POST --data '{"jsonrpc":"2.0","method":"eth_getTransactionByHash","params":["<ethereum tx>"],"id":1}'`
 
+From here, in your other available terminal, 
 And obviously more, via the Ethereum JSON-RPC API).
 
 When in doubt, you can also run the tests against a Ganache instance via `yarn test-ganache`, to make sure they are behaving correctly.

tests/solidity/package.json

@@ -11,5 +11,11 @@
     "nohoist": [
       "**/@aragon/contract-helpers-test"
     ]
+  },
+  "dependencies": {
+    "yargs": "^17.0.1"
+  },
+  "scripts": {
+    "test": "node test-helper.js"
   }
 }

tests/solidity/suites/basic/main.go

@@ -1,158 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"os"
-	"time"
-
-	"github.com/ethereum/go-ethereum/common/hexutil"
-)
-
-var (
-	HOST = os.Getenv("HOST")
-	HOME = os.Getenv("PWD")
-)
-
-type Request struct {
-	Version string      `json:"jsonrpc"`
-	Method  string      `json:"method"`
-	Params  interface{} `json:"params"`
-	ID      int         `json:"id"`
-}
-
-type RPCError struct {
-	Code    int         `json:"code"`
-	Message string      `json:"message"`
-	Data    interface{} `json:"data,omitempty"`
-}
-
-type Response struct {
-	Error  *RPCError       `json:"error"`
-	ID     int             `json:"id"`
-	Result json.RawMessage `json:"result,omitempty"`
-}
-
-func createRequest(method string, params interface{}) Request {
-	return Request{
-		Version: "2.0",
-		Method:  method,
-		Params:  params,
-		ID:      1,
-	}
-}
-
-func getTransactionReceipt(hash hexutil.Bytes) (map[string]interface{}, error) {
-	param := []string{hash.String()}
-
-	rpcRes, err := call("eth_getTransactionReceipt", param)
-	if err != nil {
-		return nil, err
-	}
-
-	receipt := make(map[string]interface{})
-
-	err = json.Unmarshal(rpcRes.Result, &receipt)
-	if err != nil {
-		return nil, err
-	}
-
-	return receipt, nil
-}
-
-func waitForReceipt(hash hexutil.Bytes) (map[string]interface{}, error) {
-	for i := 0; i < 10; i++ {
-		receipt, err := getTransactionReceipt(hash)
-		if receipt != nil {
-			return receipt, err
-		} else if err != nil {
-			return nil, err
-		}
-
-		time.Sleep(time.Second)
-	}
-
-	return nil, errors.New("cound not find transaction on chain")
-}
-
-func call(method string, params interface{}) (*Response, error) {
-	var rpcRes *Response
-
-	if HOST == "" {
-		HOST = "http://localhost:8545"
-	}
-
-	req, err := json.Marshal(createRequest(method, params))
-	if err != nil {
-		return nil, err
-	}
-
-	time.Sleep(1000000 * time.Nanosecond)
-	/* #nosec */
-	res, err := http.Post(HOST, "application/json", bytes.NewBuffer(req))
-	if err != nil {
-		return nil, err
-	}
-
-	decoder := json.NewDecoder(res.Body)
-	rpcRes = new(Response)
-
-	if err := decoder.Decode(&rpcRes); err != nil {
-		return nil, err
-	}
-
-	if err := res.Body.Close(); err != nil {
-		return nil, err
-	}
-
-	return rpcRes, nil
-}
-
-func main() {
-	var hash hexutil.Bytes
-
-	dat, err := ioutil.ReadFile(HOME + "/counter/counter_sol.bin")
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	param := make([]map[string]string, 1)
-	param[0] = make(map[string]string)
-	param[0]["from"] = os.Args[1]
-	param[0]["data"] = "0x" + string(dat)
-
-	txRPCRes, err := call("eth_sendTransaction", param)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	if err := json.Unmarshal(txRPCRes.Result, &hash); err != nil {
-		log.Fatal(err)
-	}
-
-	fmt.Println("Contract TX hash: ", hash)
-
-	receipt, err := waitForReceipt(hash)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	/*
-		//test for bad hash
-		testhash, err := hexutil.Decode("0xe146d95c74a48e730bf825c2a3dcbce8122b8a463bc15bcbb38b9c195402f0a5")
-		if err != nil {
-			log.Fatal(err)
-		}
-		receipt, err := waitForReceipt(testhash)
-		if err != nil {
-			log.Fatal(err)
-		}
-	*/
-
-	fmt.Println("receipt: ", receipt)
-}

tests/solidity/suites/basic/package.json

@@ -9,9 +9,7 @@
   },
   "devDependencies": {
     "truffle": "^5.1.42",
+    "truffle-assertions": "^0.9.2",
     "web3": "^1.2.11"
-  },
-  "dependencies": {
-    "truffle": "^5.1.43"
   }
-}
+}

tests/solidity/suites/basic/test/counter.js

@@ -1,10 +1,25 @@
 const Counter = artifacts.require("Counter")
+const truffleAssert = require('truffle-assertions');
 
-contract('Counter', ([one, two, three]) => {
+async function expectRevert(promise) {
-  console.log(one, two, three)
+  try {
+    await promise;
+  } catch (error) {
+    if (error.message.indexOf('revert') === -1) {
+      expect('revert').to.equal(error.message, 'Wrong kind of exception received');
+    }
+    return;
+  }
+  expect.fail('Expected an exception but none was received');
+}
+
+contract('Counter', (accounts) => {
+  console.log(`Using Accounts (${accounts.length}): \n${accounts.join('\n')}`);
+  console.log('==========================\n');
+  const [one, two, three] = accounts;
   let counter
 
-  beforeEach(async() => {
+  beforeEach(async () => {
     counter = await Counter.new()
     console.log('Counter:', counter.address)
 
@@ -15,7 +30,7 @@ contract('Counter', ([one, two, three]) => {
     console.log('')
   })
 
-  it('should add', async() => {
+  it('should add', async () => {
     const balanceOne = await web3.eth.getBalance(one)
     const balanceTwo = await web3.eth.getBalance(two)
     const balanceThree = await web3.eth.getBalance(three)
@@ -41,7 +56,7 @@ contract('Counter', ([one, two, three]) => {
     assert.notEqual(balanceThree, await web3.eth.getBalance(three), `${three}'s balance should be different`)
   })
 
-  it('should subtract', async() => {
+  it('should subtract', async () => {
     let count
 
     await counter.add()
@@ -68,13 +83,7 @@ contract('Counter', ([one, two, three]) => {
     assert.equal(allEvents.length, 3)
     assert.equal(changedEvents.length, 2)
 
-    try {
+    await expectRevert(counter.subtract());
-      await counter.subtract()
+
-      assert.fail('Subtracting past 0 should have reverted')
-    } catch (err) {
-      console.log()
-      console.log('Passed -- was expecting error')
-      console.log('Error:', err)
-    }
   })
 })

tests/solidity/suites/storage/contracts/EventTest.sol

@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: GPL-3.0
+
+pragma solidity >=0.7.0 <0.9.0;
+
+/**
+ * @title Storage
+ * @dev Store & retrieve value in a variable
+ */
+contract EventTest {
+
+    uint256 number;
+
+    event ValueStored1(
+        uint value1
+    );
+    event ValueStored2(
+        string msg,
+        uint value1
+    );
+    event ValueStored3(
+        string msg,
+        uint indexed value1,
+        uint value2
+    );
+
+    function store(uint256 num) public {
+        number = num;
+    }
+
+    function storeWithEvent(uint256 num) public {
+        number = num;
+        emit ValueStored1(num);
+        emit ValueStored2("TestMsg", num);
+        emit ValueStored3("TestMsg", num, num);
+    }
+
+}

tests/solidity/suites/storage/contracts/Storage.sol

@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: GPL-3.0
+
+pragma solidity >=0.7.0 <0.9.0;
+
+/**
+ * @title Storage
+ * @dev Store & retrieve value in a variable
+ */
+contract Storage {
+
+    uint256 number;
+
+    /**
+     * @dev Store value in variable
+     * @param num value to store
+     */
+    function store(uint256 num) public {
+        number = num;
+    }
+
+    /**
+     * @dev Return value 
+     * @return value of 'number'
+     */
+    function retrieve() public view returns (uint256){
+        return number;
+    }
+
+    function shouldRevert() public {
+        require(false, 'This must REVERT');
+    }
+}

tests/solidity/suites/storage/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "eth-call-test",
+  "version": "1.0.0",
+  "main": "index.js",
+  "license": "MIT",
+  "dependencies": {
+    "@truffle/hdwallet-provider": "^1.4.1",
+    "concurrently": "^6.2.0",
+    "ganache-cli": "^6.12.2",
+    "truffle": "^5.3.3",
+    "truffle-assertions": "^0.9.2"
+  },
+  "scripts": {
+    "test-ganache": "yarn truffle test",
+    "test-ethermint": "yarn truffle test --network ethermint"
+  }
+}

tests/solidity/suites/storage/test/0_test_contracts.test.js

@@ -0,0 +1,30 @@
+const Storage = artifacts.require('Storage');
+
+
+contract('Test Storage Contract', async function (accounts) {
+
+  let storageInstance;
+
+  before(function () {
+    console.log(`Using Accounts (${accounts.length}): \n${accounts.join('\n')}`);
+    console.log('==========================\n');
+  })
+
+  it('should deploy Stroage contract', async function () {
+    storageInstance = await Storage.new();
+    console.log(`Deployed Storage at: ${storageInstance.address}`);
+    expect(storageInstance.address).not.to.be.undefined;
+  });
+
+  it('should succesfully stored a value', async function () {
+    const tx = await storageInstance.store(888);
+    console.log(`Stored value 888 by tx: ${tx.tx}`);
+    expect(tx.tx).not.to.be.undefined;
+  });
+
+  it('should succesfully retrieve a value', async function () {
+    const value = await storageInstance.retrieve();
+    expect(value.toString()).to.equal('888');
+  });
+
+})

tests/solidity/suites/storage/test/1_test_evm_revert.test.js

@@ -0,0 +1,34 @@
+const Storage = artifacts.require('Storage');
+
+async function expectRevert(promise) {
+  try {
+    await promise;
+  } catch (error) {
+    if (error.message.indexOf('revert') === -1) {
+      expect('revert').to.equal(error.message, 'Wrong kind of exception received');
+    }
+    return;
+  }
+  expect.fail('Expected an exception but none was received');
+}
+
+
+contract('Test EVM Revert', async function (accounts) {
+
+  before(function () {
+    console.log(`Using Accounts (${accounts.length}): \n${accounts.join('\n')}`);
+    console.log('==========================\n');
+  })
+
+  let storageInstance;
+  it('should deploy Stroage contract', async function () {
+    storageInstance = await Storage.new();
+    console.log(`Deployed Storage at: ${storageInstance.address}`);
+    expect(storageInstance.address).not.to.be.undefined;
+  });
+
+  it('should revert when call `shouldRevert()`', async function () {
+    await expectRevert(storageInstance.shouldRevert());
+  });
+
+})

tests/solidity/suites/storage/test/2_test_events.test.js

@@ -0,0 +1,33 @@
+const EventTest = artifacts.require('EventTest');
+const truffleAssert = require('truffle-assertions');
+
+contract('Test EventTest Contract', async function (accounts) {
+
+  let eventInstance;
+
+  before(function () {
+    console.log(`Using Accounts (${accounts.length}): \n${accounts.join('\n')}`);
+    console.log('==========================\n');
+  })
+
+  it('should deploy EventTest contract', async function () {
+    eventInstance = await EventTest.new();
+    console.log(`Deployed EventTest at: ${eventInstance.address}`);
+    expect(eventInstance.address).not.to.be.undefined;
+  });
+
+  it('should emit events', async function () {
+    const tx = await eventInstance.storeWithEvent(888);
+    truffleAssert.eventEmitted(tx, 'ValueStored1', events => {
+      return events['0'].toString() === '888';
+    });
+    truffleAssert.eventEmitted(tx, 'ValueStored2', events => {
+      return events['0'].toString() === 'TestMsg' && events['1'].toString() === '888';
+    });
+    truffleAssert.eventEmitted(tx, 'ValueStored3', events => {
+      return events['0'].toString() === 'TestMsg' && events['1'].toString() === '888' && events['2'].toString() === '888';
+    });
+
+  });
+
+})

tests/solidity/suites/storage/truffle-config.js

@@ -0,0 +1,17 @@
+module.exports = {
+  networks: {
+    // Development network is just left as truffle's default settings
+    ethermint: {
+      host: "127.0.0.1",     // Localhost (default: none)
+      port: 8545,            // Standard Ethereum port (default: none)
+      network_id: "*",       // Any network (default: none)
+      gas: 5000000,          // Gas sent with each transaction
+      gasPrice: 1000000000,  // 1 gwei (in wei)
+    },
+  },
+  compilers: {
+    solc: {
+      version: "0.8.3",
+    },
+  },
+}

tests/solidity/test-helper.js

@@ -0,0 +1,187 @@
+const fs = require('fs');
+const path = require('path');
+const { exec, spawn } = require('child_process');
+const yargs = require('yargs/yargs')
+const { hideBin } = require('yargs/helpers')
+
+
+const logger = {
+  warn: msg => console.error(`WARN: ${msg}`),
+  err: msg => console.error(`ERR: ${msg}`),
+  info: msg => console.log(`INFO: ${msg}`)
+}
+
+function panic(errMsg) {
+  logger.err(errMsg);
+  process.exit(-1);
+}
+
+function checkTestEnv() {
+  
+  const argv = yargs(hideBin(process.argv))
+    .usage('Usage: $0 [options] <tests>')
+    .example('$0 --network ethermint', 'run all tests using ethermint network')
+    .example('$0 --network ethermint test1 test2', 'run only test1 and test2 using ethermint network')
+    .help('h').alias('h', 'help')
+    .describe('network', 'set which network to use: ganache|ethermint')
+    .boolean('verbose-log').describe('verbose-log', 'print ethermintd output, default false')
+    .argv;
+
+  if (!fs.existsSync(path.join(__dirname, './node_modules'))) {
+    panic('node_modules not existed. Please run `yarn install` before running tests.');
+  }
+  const runConfig = {};
+  
+  // Check test network
+  if (!argv.network) {
+    runConfig.network = 'ganache';
+  }
+  else {
+    if (argv.network !== 'ethermint' && argv.network !== 'ganache') {
+      panic('network is invalid. Must be ganache or ethermint');
+    }
+    else {
+      runConfig.network = argv.network;
+    }
+  }
+
+  // only test
+  runConfig.onlyTest = argv['_'];
+  runConfig.verboseLog = !!argv['verbose-log'];
+
+  logger.info(`Running on network: ${runConfig.network}`);
+  return runConfig;
+
+}
+
+function loadTests() {
+  const validTests = [];
+  fs.readdirSync(path.join(__dirname, 'suites')).forEach(dirname => {
+    const dirStat = fs.statSync(path.join(__dirname, 'suites', dirname));
+    if (!dirStat.isDirectory) {
+      logger.warn(`${dirname} is not a directory. Skip this test suite.`);
+      return;
+    }
+
+    const needFiles = ['package.json', 'test'];
+    for (const f of needFiles) {
+      if (!fs.existsSync(path.join(__dirname, 'suites', dirname, f))) {
+        logger.warn(`${dirname} does not contains file/dir: ${f}. Skip this test suite.`);
+        return;
+      }
+    }
+
+    // test package.json
+    try {
+      const testManifest = JSON.parse(fs.readFileSync(path.join(__dirname, 'suites', dirname, 'package.json'), 'utf-8'))
+      const needScripts = ['test-ganache', 'test-ethermint'];
+      for (const s of needScripts) {
+        if (Object.keys(testManifest['scripts']).indexOf(s) === -1) {
+          logger.warn(`${dirname} does not have test script: \`${s}\`. Skip this test suite.`);
+          return;
+        }
+      }
+    } catch (error) {
+      logger.warn(`${dirname} test package.json load failed. Skip this test suite.`);
+      logger.err(error);
+      return;
+    }
+    validTests.push(dirname);
+  })
+  return validTests;
+}
+
+function performTestSuite({ testName, network }) {
+  const cmd = network === 'ganache' ? 'test-ganache' : 'test-ethermint';
+  return new Promise((resolve, reject) => {
+    const testProc = spawn('yarn', [cmd], {
+      cwd: path.join(__dirname, 'suites', testName)
+    });
+  
+    testProc.stdout.pipe(process.stdout);
+    testProc.stderr.pipe(process.stderr);
+  
+    testProc.on('close', code => {
+      if (code === 0) {
+        console.log("end");
+        resolve();
+      }
+      else {
+        reject(new Error(`Test: ${testName} exited with error code ${code}`));
+      }
+    });
+  });
+}
+
+async function performTests({ allTests, runConfig }) {
+
+  if (allTests.length === 0) {
+    panic('No tests are found or all invalid!');
+  }
+  if (runConfig.onlyTest.length === 0) {
+    logger.info('Start all tests:');
+  }
+  else {
+    allTests = allTests.filter(t => runConfig.onlyTest.indexOf(t) !== -1);
+    logger.info(`Only run tests: (${allTests.join(', ')})`);
+  }
+
+  for (const currentTestName of allTests) {
+    logger.info(`Start test: ${currentTestName}`);
+    await performTestSuite({ testName: currentTestName, network: runConfig.network });
+  }
+
+  logger.info(`${allTests.length} test suites passed!`);
+}
+
+function setupNetwork({ runConfig, timeout }) {
+  if (runConfig.network !== 'ethermint') {
+    // no need to start ganache. Truffle will start it
+    return;
+  }
+
+  // Spawn the ethermint process
+
+  const spawnPromise = new Promise((resolve, reject) => {
+    const ethermintdProc = spawn('./init-test-node.sh', {
+      cwd: __dirname,
+      stdio: ['ignore', runConfig.verboseLog ? 'pipe' : 'ignore', 'pipe'],
+    });
+
+    logger.info(`Starting Ethermintd process... timeout: ${timeout}ms`);
+    if (runConfig.verboseLog) {
+      ethermintdProc.stdout.pipe(process.stdout);
+    }
+    ethermintdProc.stderr.on('data', d => {
+      const oLine = d.toString();
+      if (runConfig.verboseLog) {
+        process.stdout.write(oLine);
+      }
+      if (oLine.indexOf('Starting EVM RPC server') !== -1) {
+        logger.info('Ethermintd started');
+        resolve(ethermintdProc);
+      }
+    });
+  });
+
+  const timeoutPromise = new Promise((resolve, reject) => {
+    setTimeout(() => reject(new Error('Start ethermintd timeout!')), timeout);
+  });
+  return Promise.race([spawnPromise, timeoutPromise]);
+}
+
+async function main() {
+
+  const runConfig = checkTestEnv();
+  const allTests = loadTests(runConfig);
+
+  const proc = await setupNetwork({ runConfig, timeout: 50000 });
+  await performTests({ allTests, runConfig });
+
+  if (proc) {
+    proc.kill();
+  }
+  process.exit(0);
+}
+
+main();