diff --git a/cmake/EthUtils.cmake b/cmake/EthUtils.cmake
index a473abcb7..fca41107e 100644
--- a/cmake/EthUtils.cmake
+++ b/cmake/EthUtils.cmake
@@ -34,3 +34,17 @@ macro(eth_default_option O DEF)
 	endif()
 endmacro()
 
+function(detect_stray_source_files FILELIST DIRECTORY)
+	if(CMAKE_VERSION VERSION_LESS 3.12)
+		file(GLOB sources RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "${DIRECTORY}/*.cpp" "${DIRECTORY}/*.h")
+	else()
+		file(GLOB sources RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" CONFIGURE_DEPENDS "${DIRECTORY}/*.cpp" "${DIRECTORY}/*.h")
+	endif()
+	foreach(path IN LISTS FILELIST)
+		list(REMOVE_ITEM sources ${path})
+	endforeach()
+	list(LENGTH sources leftover_sources)
+	if (leftover_sources)
+		message(SEND_ERROR "The following source files are present but are not compiled: ${sources}")
+	endif()
+endfunction(detect_stray_source_files)
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index eb72ae0be..387d6e515 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -1,34 +1,159 @@
-file(GLOB sources "*.cpp")
-file(GLOB headers "*.h")
+set(sources
+    boostTest.cpp
+    Common.cpp
+    Common.h
+    EVMHost.cpp
+    EVMHost.h
+    ExecutionFramework.cpp
+    ExecutionFramework.h
+    InteractiveTests.h
+    Metadata.cpp
+    Metadata.h
+    Options.cpp
+    Options.h
+    TestCase.cpp
+    TestCase.h
+)
+detect_stray_source_files("${sources}" ".")
 
-file(GLOB contracts_sources "contracts/*.cpp")
-file(GLOB contracts_headers "contracts/*.h")
-file(GLOB libdevcore_sources "libdevcore/*.cpp")
-file(GLOB libdevcore_headers "libdevcore/*.h")
-file(GLOB liblangutil_sources "liblangutil/*.cpp")
-file(GLOB liblangutil_headers "liblangutil/*.h")
-file(GLOB libevmasm_sources "libevmasm/*.cpp")
-file(GLOB libevmasm_headers "libevmasm/*.h")
-file(GLOB libyul_sources "libyul/*.cpp")
-file(GLOB libyul_headers "libyul/*.h")
-if (LLL)
-    file(GLOB liblll_sources "liblll/*.cpp")
-    file(GLOB liblll_headers "liblll/*.h")
-endif()
-file(GLOB libsolidity_sources "libsolidity/*.cpp")
-file(GLOB libsolidity_headers "libsolidity/*.h")
-file(GLOB libsolidity_util_sources "libsolidity/util/*.cpp")
-file(GLOB libsolidity_util_headers "libsolidity/util/*.h")
+set(contracts_sources
+    contracts/AuctionRegistrar.cpp
+    contracts/ContractInterface.h
+    contracts/FixedFeeRegistrar.cpp
+    contracts/Wallet.cpp
+)
+detect_stray_source_files("${contracts_sources}" "contracts/")
 
-add_executable(soltest ${sources} ${headers}
-    ${contracts_sources} ${contracts_headers}
-    ${libdevcore_sources} ${libdevcore_headers}
-    ${liblangutil_sources} ${liblangutil_headers}
-    ${libevmasm_sources} ${libevmasm_headers}
-    ${libyul_sources} ${libyul_headers}
-    ${liblll_sources} ${liblll_headers}
-    ${libsolidity_sources} ${libsolidity_headers}
-    ${libsolidity_util_sources} ${libsolidity_util_headers}
+set(libdevcore_sources
+    libdevcore/Checksum.cpp
+    libdevcore/CommonData.cpp
+    libdevcore/IndentedWriter.cpp
+    libdevcore/IpfsHash.cpp
+    libdevcore/IterateReplacing.cpp
+    libdevcore/JSON.cpp
+    libdevcore/Keccak256.cpp
+    libdevcore/StringUtils.cpp
+    libdevcore/SwarmHash.cpp
+    libdevcore/UTF8.cpp
+    libdevcore/Whiskers.cpp
+)
+detect_stray_source_files("${libdevcore_sources}" "libdevcore/")
+
+set(libevmasm_sources
+    libevmasm/Assembler.cpp
+    libevmasm/Optimiser.cpp
+)
+detect_stray_source_files("${libevmasm_sources}" "libevmasm/")
+
+set(liblangutil_sources
+    liblangutil/CharStream.cpp
+    liblangutil/SourceLocation.cpp
+)
+detect_stray_source_files("${liblangutil_sources}" "liblangutil/")
+
+if(LLL)
+    set (liblll_sources
+        liblll/Compiler.cpp
+        liblll/EndToEndTest.cpp
+        liblll/ExecutionFramework.cpp
+        liblll/ExecutionFramework.h
+        liblll/LLL_ENS.cpp
+        liblll/LLL_ERC20.cpp
+        liblll/Parser.cpp
+    )
+    detect_stray_source_files("${liblll_sources}" "liblll/")
+endif(LLL)
+
+set(libsolidity_sources
+    libsolidity/ABIDecoderTests.cpp
+    libsolidity/ABIEncoderTests.cpp
+    libsolidity/ABIJsonTest.cpp
+    libsolidity/ABIJsonTest.h
+    libsolidity/ABITestsCommon.h
+    libsolidity/AnalysisFramework.cpp
+    libsolidity/AnalysisFramework.h
+    libsolidity/Assembly.cpp
+    libsolidity/ASTJSONTest.cpp
+    libsolidity/ASTJSONTest.h
+    libsolidity/ErrorCheck.cpp
+    libsolidity/ErrorCheck.h
+    libsolidity/GasCosts.cpp
+    libsolidity/GasMeter.cpp
+    libsolidity/GasTest.cpp
+    libsolidity/GasTest.h
+    libsolidity/Imports.cpp
+    libsolidity/InlineAssembly.cpp
+    libsolidity/LibSolc.cpp
+    libsolidity/Metadata.cpp
+    libsolidity/SemanticTest.cpp
+    libsolidity/SemanticTest.h
+    libsolidity/SemVerMatcher.cpp
+    libsolidity/SMTChecker.cpp
+    libsolidity/SMTCheckerJSONTest.cpp
+    libsolidity/SMTCheckerJSONTest.h
+    libsolidity/SolidityCompiler.cpp
+    libsolidity/SolidityEndToEndTest.cpp
+    libsolidity/SolidityExecutionFramework.cpp
+    libsolidity/SolidityExecutionFramework.h
+    libsolidity/SolidityExpressionCompiler.cpp
+    libsolidity/SolidityNameAndTypeResolution.cpp
+    libsolidity/SolidityNatspecJSON.cpp
+    libsolidity/SolidityOptimizer.cpp
+    libsolidity/SolidityParser.cpp
+    libsolidity/SolidityScanner.cpp
+    libsolidity/SolidityTypes.cpp
+    libsolidity/StandardCompiler.cpp
+    libsolidity/SyntaxTest.cpp
+    libsolidity/SyntaxTest.h
+    libsolidity/ViewPureChecker.cpp
+)
+detect_stray_source_files("${libsolidity_sources}" "libsolidity/")
+
+set(libsolidity_util_sources
+    libsolidity/util/BytesUtils.cpp
+    libsolidity/util/BytesUtils.h
+    libsolidity/util/ContractABIUtils.cpp
+    libsolidity/util/ContractABIUtils.h
+    libsolidity/util/SoltestErrors.h
+    libsolidity/util/SoltestTypes.h
+    libsolidity/util/TestFileParser.cpp
+    libsolidity/util/TestFileParser.h
+    libsolidity/util/TestFileParserTests.cpp
+    libsolidity/util/TestFunctionCall.cpp
+    libsolidity/util/TestFunctionCall.h
+    libsolidity/util/TestFunctionCallTests.cpp
+)
+detect_stray_source_files("${libsolidity_util_sources}" "libsolidity/util/")
+
+set(libyul_sources
+    libyul/Common.cpp
+    libyul/Common.h
+    libyul/CompilabilityChecker.cpp
+    libyul/FunctionSideEffects.cpp
+    libyul/FunctionSideEffects.h
+    libyul/Inliner.cpp
+    libyul/Metrics.cpp
+    libyul/ObjectCompilerTest.cpp
+    libyul/ObjectCompilerTest.h
+    libyul/ObjectParser.cpp
+    libyul/Parser.cpp
+    libyul/StackReuseCodegen.cpp
+    libyul/YulInterpreterTest.cpp
+    libyul/YulInterpreterTest.h
+    libyul/YulOptimizerTest.cpp
+    libyul/YulOptimizerTest.h
+)
+detect_stray_source_files("${libyul_sources}" "libyul/")
+
+add_executable(soltest ${sources}
+    ${contracts_sources}
+    ${libdevcore_sources}
+    ${liblangutil_sources}
+    ${libevmasm_sources}
+    ${libyul_sources}
+    ${liblll_sources}
+    ${libsolidity_sources}
+    ${libsolidity_util_sources}
 )
 target_link_libraries(soltest PRIVATE libsolc yul solidity yulInterpreter evmasm devcore Boost::boost Boost::program_options Boost::unit_test_framework evmc)