Merge pull request #12283 from ethereum/soltest-graceful-error-handling

Graceful error handling in soltest/isoltest + improved soltestAssert()

liblangutil/Exceptions.h

@@ -58,13 +58,13 @@ struct InvalidAstError: virtual util::Exception {};
 #endif
 
 #define solAssert_1(CONDITION) \
-	solAssert_2(CONDITION, "")
+	solAssert_2((CONDITION), "")
 
 #define solAssert_2(CONDITION, DESCRIPTION) \
 	assertThrowWithDefaultDescription( \
-		CONDITION, \
+		(CONDITION), \
 		::solidity::langutil::InternalCompilerError, \
-		DESCRIPTION, \
+		(DESCRIPTION), \
 		"Solidity assertion failed" \
 	)
 
@@ -77,13 +77,13 @@ struct InvalidAstError: virtual util::Exception {};
 #endif
 
 #define solUnimplementedAssert_1(CONDITION) \
-	solUnimplementedAssert_2(CONDITION, "")
+	solUnimplementedAssert_2((CONDITION), "")
 
 #define solUnimplementedAssert_2(CONDITION, DESCRIPTION) \
 	assertThrowWithDefaultDescription( \
-		CONDITION, \
+		(CONDITION), \
 		::solidity::langutil::UnimplementedFeatureError, \
-		DESCRIPTION, \
+		(DESCRIPTION), \
 		"Unimplemented feature" \
 	)
 
@@ -105,9 +105,9 @@ struct InvalidAstError: virtual util::Exception {};
 
 #define astAssert_2(CONDITION, DESCRIPTION) \
 	assertThrowWithDefaultDescription( \
-		CONDITION, \
+		(CONDITION), \
 		::solidity::langutil::InvalidAstError, \
-		DESCRIPTION, \
+		(DESCRIPTION), \
 		"AST assertion failed" \
 	)
 

libsmtutil/Exceptions.h

@@ -38,13 +38,13 @@ struct SMTLogicError: virtual util::Exception {};
 #endif
 
 #define smtAssert_1(CONDITION) \
-	smtAssert_2(CONDITION, "")
+	smtAssert_2((CONDITION), "")
 
 #define smtAssert_2(CONDITION, DESCRIPTION) \
 	assertThrowWithDefaultDescription( \
-		CONDITION, \
+		(CONDITION), \
 		::solidity::smtutil::SMTLogicError, \
-		DESCRIPTION, \
+		(DESCRIPTION), \
 		"SMT assertion failed" \
 	)
 

libsolutil/Assertions.h

@@ -63,7 +63,7 @@ inline std::string stringOrDefault(std::string _string, std::string _defaultStri
 		if (!(_condition)) \
 			solThrow( \
 				_exceptionType, \
-				::solidity::util::assertions::stringOrDefault(_description, _defaultDescription) \
+				::solidity::util::assertions::stringOrDefault((_description), (_defaultDescription)) \
 			); \
 	} \
 	while (false)
@@ -72,6 +72,6 @@ inline std::string stringOrDefault(std::string _string, std::string _defaultStri
 /// Use it as assertThrow(1 == 1, ExceptionType, "Mathematics is wrong.");
 /// The second parameter must be an exception class (rather than an instance).
 #define assertThrow(_condition, _exceptionType, _description) \
-	assertThrowWithDefaultDescription(_condition, _exceptionType, _description, "Assertion failed")
+	assertThrowWithDefaultDescription((_condition), _exceptionType, (_description), "Assertion failed")
 
 }

libsolutil/Exceptions.h

@@ -48,7 +48,7 @@ struct Exception: virtual std::exception, virtual boost::exception
 #define solThrow(_exceptionType, _description) \
 	::boost::throw_exception( \
 		_exceptionType() << \
-		::solidity::util::errinfo_comment(_description) << \
+		::solidity::util::errinfo_comment((_description)) << \
 		::boost::throw_function(ETH_FUNC) << \
 		::boost::throw_file(__FILE__) << \
 		::boost::throw_line(__LINE__) \

libyul/Exceptions.h

@@ -63,13 +63,13 @@ struct StackTooDeepError: virtual YulException
 #endif
 
 #define yulAssert_1(CONDITION) \
-	yulAssert_2(CONDITION, "")
+	yulAssert_2((CONDITION), "")
 
 #define yulAssert_2(CONDITION, DESCRIPTION) \
 	assertThrowWithDefaultDescription( \
-		CONDITION, \
+		(CONDITION), \
 		::solidity::yul::YulAssertion, \
-		DESCRIPTION, \
+		(DESCRIPTION), \
 		"Yul assertion failed" \
 	)
 

test/Common.cpp

@@ -159,6 +159,8 @@ bool CommonOptions::parse(int argc, char const* const* argv)
 	po::variables_map arguments;
 	addOptions();
 
+	try
+	{
 		po::command_line_parser cmdLineParser(argc, argv);
 		cmdLineParser.options(options);
 		auto parsedOptions = cmdLineParser.run();
@@ -179,6 +181,11 @@ bool CommonOptions::parse(int argc, char const* const* argv)
 					errorMessage << token;
 				BOOST_THROW_EXCEPTION(std::runtime_error(errorMessage.str()));
 			}
+	}
+	catch (po::error const& exception)
+	{
+		solThrow(ConfigException, exception.what());
+	}
 
 	if (vmPaths.empty())
 	{

test/Common.h

@@ -75,6 +75,9 @@ struct CommonOptions
 	langutil::EVMVersion evmVersion() const;
 
 	virtual void addOptions();
+	// @returns true if the program should continue, false if it should exit immediately without
+	// reporting an error.
+	// Throws ConfigException or std::runtime_error if parsing fails.
 	virtual bool parse(int argc, char const* const* argv);
 	// Throws a ConfigException on error
 	virtual void validate() const;

test/boostTest.cpp

@@ -204,15 +204,18 @@ int registerTests(
 	return numTestsAdded;
 }
 
-void initializeOptions()
+bool initializeOptions()
 {
 	auto const& suite = boost::unit_test::framework::master_test_suite();
 
 	auto options = std::make_unique<solidity::test::CommonOptions>();
-	solAssert(options->parse(suite.argc, suite.argv), "Failed to parse options!");
+	bool shouldContinue = options->parse(suite.argc, suite.argv);
+	if (!shouldContinue)
+		return false;
 	options->validate();
 
 	solidity::test::CommonOptions::setSingleton(std::move(options));
+	return true;
 }
 
 }
@@ -228,10 +231,14 @@ test_suite* init_unit_test_suite( int /*argc*/, char* /*argv*/[] )
 	master_test_suite_t& master = framework::master_test_suite();
 	master.p_name.value = "SolidityTests";
 
-	initializeOptions();
+	try
+	{
+		bool shouldContinue = initializeOptions();
+		if (!shouldContinue)
+			exit(EXIT_SUCCESS);
 
 		if (!solidity::test::loadVMs(solidity::test::CommonOptions::get()))
-		exit(1);
+			exit(EXIT_FAILURE);
 
 		if (solidity::test::CommonOptions::get().disableSemanticTests)
 			cout << endl << "--- SKIPPING ALL SEMANTICS TESTS ---" << endl << endl;
@@ -287,6 +294,17 @@ test_suite* init_unit_test_suite( int /*argc*/, char* /*argv*/[] )
 			})
 				removeTestSuite(suite);
 		}
+	}
+	catch (solidity::test::ConfigException const& exception)
+	{
+		cerr << exception.what() << endl;
+		exit(EXIT_FAILURE);
+	}
+	catch (std::runtime_error const& exception)
+	{
+		cerr << exception.what() << endl;
+		exit(EXIT_FAILURE);
+	}
 
 	return nullptr;
 }

test/libsolidity/util/SoltestErrors.h

@@ -15,20 +15,35 @@
 #pragma once
 
 #include <libsolutil/AnsiColorized.h>
+#include <libsolutil/Assertions.h>
 #include <libsolutil/CommonData.h>
 #include <libsolutil/Exceptions.h>
 
+#include <boost/preprocessor/cat.hpp>
+#include <boost/preprocessor/facilities/empty.hpp>
+#include <boost/preprocessor/facilities/overload.hpp>
+
 namespace solidity::frontend::test
 {
 
-#define soltestAssert(CONDITION, DESCRIPTION) \
-	do \
-	{ \
-		if (!(CONDITION)) \
-			BOOST_THROW_EXCEPTION(std::runtime_error(DESCRIPTION)); \
-	} \
-	while (false)
+struct InternalSoltestError: virtual util::Exception {};
 
+#if !BOOST_PP_VARIADICS_MSVC
+#define soltestAssert(...) BOOST_PP_OVERLOAD(soltestAssert_,__VA_ARGS__)(__VA_ARGS__)
+#else
+#define soltestAssert(...) BOOST_PP_CAT(BOOST_PP_OVERLOAD(soltestAssert_,__VA_ARGS__)(__VA_ARGS__),BOOST_PP_EMPTY())
+#endif
+
+#define soltestAssert_1(CONDITION) \
+	soltestAssert_2((CONDITION), "")
+
+#define soltestAssert_2(CONDITION, DESCRIPTION) \
+	assertThrowWithDefaultDescription( \
+		(CONDITION), \
+		::solidity::frontend::test::InternalSoltestError, \
+		(DESCRIPTION), \
+		"Soltest assertion failed" \
+	)
 
 class TestParserError: virtual public util::Exception
 {

test/tools/IsolTestOptions.cpp

@@ -75,9 +75,9 @@ void IsolTestOptions::addOptions()
 
 bool IsolTestOptions::parse(int _argc, char const* const* _argv)
 {
-	bool const res = CommonOptions::parse(_argc, _argv);
+	bool const shouldContinue = CommonOptions::parse(_argc, _argv);
 
-	if (showHelp || !res)
+	if (showHelp || !shouldContinue)
 	{
 		std::cout << options << std::endl;
 		return false;
@@ -85,7 +85,7 @@ bool IsolTestOptions::parse(int _argc, char const* const* _argv)
 
 	enforceGasTest = enforceGasTest || (evmVersion() == langutil::EVMVersion{} && !useABIEncoderV1);
 
-	return res;
+	return shouldContinue;
 }
 
 void IsolTestOptions::validate() const

test/tools/isoltest.cpp

@@ -433,8 +433,9 @@ int main(int argc, char const *argv[])
 		{
 			auto options = std::make_unique<IsolTestOptions>();
 
-			if (!options->parse(argc, argv))
-				return -1;
+			bool shouldContinue = options->parse(argc, argv);
+			if (!shouldContinue)
+				return EXIT_SUCCESS;
 
 			options->validate();
 			CommonOptions::setSingleton(std::move(options));
@@ -443,7 +444,7 @@ int main(int argc, char const *argv[])
 		auto& options = dynamic_cast<IsolTestOptions const&>(CommonOptions::get());
 
 		if (!solidity::test::loadVMs(options))
-			return 1;
+			return EXIT_FAILURE;
 
 		if (options.disableSemanticTests)
 			cout << endl << "--- SKIPPING ALL SEMANTICS TESTS ---" << endl << endl;
@@ -479,7 +480,7 @@ int main(int argc, char const *argv[])
 			if (stats)
 				global_stats += *stats;
 			else
-				return 1;
+				return EXIT_FAILURE;
 		}
 
 		cout << endl << "Summary: ";
@@ -497,11 +498,22 @@ int main(int argc, char const *argv[])
 		if (options.disableSemanticTests)
 			cout << "\nNOTE: Skipped semantics tests.\n" << endl;
 
-		return global_stats ? 0 : 1;
+		return global_stats ? EXIT_SUCCESS : EXIT_FAILURE;
 	}
-	catch (std::exception const& _exception)
+	catch (boost::program_options::error const& exception)
 	{
-		cerr << _exception.what() << endl;
-		return 1;
+		cerr << exception.what() << endl;
+		return EXIT_FAILURE;
+	}
+	catch (std::runtime_error const& exception)
+	{
+		cerr << exception.what() << endl;
+		return EXIT_FAILURE;
+	}
+	catch (...)
+	{
+		cerr << "Unhandled exception caught." << endl;
+		cerr << boost::current_exception_diagnostic_information() << endl;
+		return EXIT_FAILURE;
 	}
 }