From 52dfccca98d870566c764204495582a1656b3a10 Mon Sep 17 00:00:00 2001 From: wechman Date: Fri, 4 Mar 2022 10:15:57 +0100 Subject: [PATCH] Replace all locale-dependent operations with locale-agnostic counterparts --- libevmasm/AssemblyItem.cpp | 4 +- liblangutil/Token.cpp | 11 +++-- libsolidity/ast/Types.cpp | 5 ++- .../codegen/ir/IRGeneratorForStatements.cpp | 2 +- libsolidity/formal/CHC.cpp | 5 ++- libsolutil/CommonData.cpp | 11 ++--- libsolutil/StringUtils.h | 45 +++++++++++++++++++ libyul/backends/evm/EVMDialect.cpp | 20 ++++----- libyul/optimiser/SimplificationRules.cpp | 14 +++--- test/CommonSyntaxTest.cpp | 13 +++--- test/TestCase.cpp | 2 +- test/TestCase.h | 2 +- test/libsolidity/util/BytesUtils.cpp | 3 +- test/libsolidity/util/TestFileParser.cpp | 15 ++++--- test/tools/ossfuzz/strictasm_diff_ossfuzz.cpp | 3 +- test/tools/yulopti.cpp | 3 +- 16 files changed, 106 insertions(+), 52 deletions(-) diff --git a/libevmasm/AssemblyItem.cpp b/libevmasm/AssemblyItem.cpp index eeeadb3ef..984058765 100644 --- a/libevmasm/AssemblyItem.cpp +++ b/libevmasm/AssemblyItem.cpp @@ -200,9 +200,7 @@ string AssemblyItem::toAssemblyText(Assembly const& _assembly) const case Operation: { assertThrow(isValidInstruction(instruction()), AssemblyException, "Invalid instruction."); - string name = instructionInfo(instruction()).name; - transform(name.begin(), name.end(), name.begin(), [](unsigned char _c) { return tolower(_c); }); - text = name; + text = util::toLower(instructionInfo(instruction()).name); break; } case Push: diff --git a/liblangutil/Token.cpp b/liblangutil/Token.cpp index 8bdaff1fd..d7620ef13 100644 --- a/liblangutil/Token.cpp +++ b/liblangutil/Token.cpp @@ -40,10 +40,13 @@ // You should have received a copy of the GNU General Public License // along with solidity. If not, see . -#include #include +#include +#include + #include + using namespace std; namespace solidity::langutil @@ -180,11 +183,11 @@ tuple fromIdentifierOrKeyword(string const& _ return ret; }; - auto positionM = find_if(_literal.begin(), _literal.end(), ::isdigit); + auto positionM = find_if(_literal.begin(), _literal.end(), util::isDigit); if (positionM != _literal.end()) { string baseType(_literal.begin(), positionM); - auto positionX = find_if_not(positionM, _literal.end(), ::isdigit); + auto positionX = find_if_not(positionM, _literal.end(), util::isDigit); int m = parseSize(positionM, positionX); Token keyword = keywordByName(baseType); if (keyword == Token::Bytes) @@ -208,7 +211,7 @@ tuple fromIdentifierOrKeyword(string const& _ positionM < positionX && positionX < _literal.end() && *positionX == 'x' && - all_of(positionX + 1, _literal.end(), ::isdigit) + all_of(positionX + 1, _literal.end(), util::isDigit) ) { int n = parseSize(positionX + 1, _literal.end()); if ( diff --git a/libsolidity/ast/Types.cpp b/libsolidity/ast/Types.cpp index 8a6905307..c84d79785 100644 --- a/libsolidity/ast/Types.cpp +++ b/libsolidity/ast/Types.cpp @@ -33,6 +33,7 @@ #include #include #include +#include #include #include @@ -781,8 +782,8 @@ tuple RationalNumberType::parseRational(string const& _value) if (radixPoint != _value.end()) { if ( - !all_of(radixPoint + 1, _value.end(), ::isdigit) || - !all_of(_value.begin(), radixPoint, ::isdigit) + !all_of(radixPoint + 1, _value.end(), util::isDigit) || + !all_of(_value.begin(), radixPoint, util::isDigit) ) return make_tuple(false, rational(0)); diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp index ed49c94b0..01c012e65 100644 --- a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp +++ b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp @@ -203,7 +203,7 @@ private: else solAssert(false); - if (isdigit(value.front())) + if (isDigit(value.front())) return yul::Literal{_identifier.debugData, yul::LiteralKind::Number, yul::YulString{value}, {}}; else return yul::Identifier{_identifier.debugData, yul::YulString{value}}; diff --git a/libsolidity/formal/CHC.cpp b/libsolidity/formal/CHC.cpp index 9e31d7fc9..cbadc2c1d 100644 --- a/libsolidity/formal/CHC.cpp +++ b/libsolidity/formal/CHC.cpp @@ -33,6 +33,7 @@ #include #include #include +#include #ifdef HAVE_Z3_DLOPEN #include @@ -1998,9 +1999,9 @@ map> CHC::summaryCalls(CHCSolverInterface::CexGraph c // Predicates that do not have a CALLID have a predicate id at the end of , // so the assertion below should still hold. auto beg = _s.data(); - while (beg != _s.data() + _s.size() && !isdigit(*beg)) ++beg; + while (beg != _s.data() + _s.size() && !isDigit(*beg)) ++beg; auto end = beg; - while (end != _s.data() + _s.size() && isdigit(*end)) ++end; + while (end != _s.data() + _s.size() && isDigit(*end)) ++end; solAssert(beg != end, "Expected to find numerical call or predicate id."); diff --git a/libsolutil/CommonData.cpp b/libsolutil/CommonData.cpp index a49f90491..437a090c2 100644 --- a/libsolutil/CommonData.cpp +++ b/libsolutil/CommonData.cpp @@ -20,11 +20,12 @@ * @date 2014 */ +#include #include #include -#include -#include #include +#include +#include #include @@ -154,9 +155,9 @@ string solidity::util::getChecksummedAddress(string const& _addr) char addressCharacter = s[i]; uint8_t nibble = hash[i / 2u] >> (4u * (1u - (i % 2u))) & 0xf; if (nibble >= 8) - ret += static_cast(toupper(addressCharacter)); + ret += toUpper(addressCharacter); else - ret += static_cast(tolower(addressCharacter)); + ret += toLower(addressCharacter); } return ret; } @@ -211,7 +212,7 @@ string solidity::util::escapeAndQuoteString(string const& _input) out += "\\r"; else if (c == '\t') out += "\\t"; - else if (!isprint(c, locale::classic())) + else if (!isPrint(c)) { ostringstream o; o << "\\x" << std::hex << setfill('0') << setw(2) << (unsigned)(unsigned char)(c); diff --git a/libsolutil/StringUtils.h b/libsolutil/StringUtils.h index 09aa30be9..7a48c296f 100644 --- a/libsolutil/StringUtils.h +++ b/libsolutil/StringUtils.h @@ -27,7 +27,9 @@ #include #include +#include #include +#include #include #include @@ -197,4 +199,47 @@ inline std::optional toUnsignedInt(std::string const& _value) } } +/// Converts parameter _c to its lowercase equivalent if c is an uppercase letter and has a lowercase equivalent. It uses the classic "C" locale semantics. +/// @param _c value to be converted +/// @return the converted value +inline char toLower(char _c) +{ + return tolower(_c, std::locale::classic()); +} + +/// Converts parameter _c to its uppercase equivalent if c is an lowercase letter and has a uppercase equivalent. It uses the classic "C" locale semantics. +/// @param _c value to be converted +/// @return the converted value +inline char toUpper(char _c) +{ + return toupper(_c, std::locale::classic()); +} + +/// Converts parameter _s to its lowercase equivalent. It uses the classic "C" locale semantics. +/// @param _s value to be converted +/// @return the converted value +inline std::string toLower(std::string _s) +{ + std::transform(_s.begin(), _s.end(), _s.begin(), [](char _c) { + return toLower(_c); + }); + return _s; +} + +/// Checks whether _c is a decimal digit character. It uses the classic "C" locale semantics. +/// @param _c character to be checked +/// @return true if _c is a decimal digit character, false otherwise +inline bool isDigit(char _c) +{ + return isdigit(_c, std::locale::classic()); +} + +// Checks if character is printable using classic "C" locale +/// @param _c character to be checked +/// @return true if _c is a printable character, false otherwise. +inline bool isPrint(char _c) +{ + return isprint(_c, std::locale::classic()); +} + } diff --git a/libyul/backends/evm/EVMDialect.cpp b/libyul/backends/evm/EVMDialect.cpp index 0fefb24d0..35fa5f88e 100644 --- a/libyul/backends/evm/EVMDialect.cpp +++ b/libyul/backends/evm/EVMDialect.cpp @@ -21,17 +21,17 @@ #include -#include +#include +#include +#include +#include #include -#include -#include +#include #include +#include +#include #include #include -#include -#include - -#include #include #include @@ -121,8 +121,7 @@ set createReservedIdentifiers(langutil::EVMVersion _evmVersion) set reserved; for (auto const& instr: evmasm::c_instructions) { - string name = instr.first; - transform(name.begin(), name.end(), name.begin(), [](unsigned char _c) { return tolower(_c); }); + string name = toLower(instr.first); if (!baseFeeException(instr.second)) reserved.emplace(name); } @@ -142,8 +141,7 @@ map createBuiltins(langutil::EVMVersion _evmVe map builtins; for (auto const& instr: evmasm::c_instructions) { - string name = instr.first; - transform(name.begin(), name.end(), name.begin(), [](unsigned char _c) { return tolower(_c); }); + string name = toLower(instr.first); auto const opcode = instr.second; if ( diff --git a/libyul/optimiser/SimplificationRules.cpp b/libyul/optimiser/SimplificationRules.cpp index 55c30d2f2..3e906fcdf 100644 --- a/libyul/optimiser/SimplificationRules.cpp +++ b/libyul/optimiser/SimplificationRules.cpp @@ -21,15 +21,16 @@ #include -#include -#include -#include -#include -#include #include #include +#include +#include +#include +#include +#include #include +#include using namespace std; using namespace solidity; @@ -249,8 +250,7 @@ Expression Pattern::toExpression(shared_ptr const& _debugData) for (auto const& arg: m_arguments) arguments.emplace_back(arg.toExpression(_debugData)); - string name = instructionInfo(m_instruction).name; - transform(begin(name), end(name), begin(name), [](auto _c) { return tolower(_c); }); + string name = util::toLower(instructionInfo(m_instruction).name); return FunctionCall{_debugData, Identifier{_debugData, YulString{name}}, diff --git a/test/CommonSyntaxTest.cpp b/test/CommonSyntaxTest.cpp index 11f654281..d28ec1675 100644 --- a/test/CommonSyntaxTest.cpp +++ b/test/CommonSyntaxTest.cpp @@ -29,7 +29,6 @@ using namespace std; using namespace solidity; -using namespace solidity::util; using namespace solidity::util::formatting; using namespace solidity::langutil; using namespace solidity::frontend; @@ -43,10 +42,11 @@ namespace int parseUnsignedInteger(string::iterator& _it, string::iterator _end) { - if (_it == _end || !isdigit(*_it)) + auto isDigit = [](char _c) -> bool {return isdigit(_c, std::locale::classic());}; + if (_it == _end || !isDigit(*_it)) BOOST_THROW_EXCEPTION(runtime_error("Invalid test expectation. Source location expected.")); int result = 0; - while (_it != _end && isdigit(*_it)) + while (_it != _end && isDigit(*_it)) { result *= 10; result += *_it - '0'; @@ -195,6 +195,7 @@ string CommonSyntaxTest::errorMessage(Exception const& _e) vector CommonSyntaxTest::parseExpectations(istream& _stream) { + auto isDigit = [](char _c) -> bool {return isdigit(_c, std::locale::classic());}; vector expectations; string line; while (getline(_stream, line)) @@ -207,14 +208,14 @@ vector CommonSyntaxTest::parseExpectations(istream& _stream) if (it == line.end()) continue; auto typeBegin = it; - while (it != line.end() && isalpha(*it)) + while (it != line.end() && isalpha(*it, locale::classic())) ++it; string errorType(typeBegin, it); skipWhitespace(it, line.end()); optional errorId; - if (it != line.end() && isdigit(*it)) + if (it != line.end() && isDigit(*it)) errorId = ErrorId{static_cast(parseUnsignedInteger(it, line.end()))}; expect(it, line.end(), ':'); @@ -227,7 +228,7 @@ vector CommonSyntaxTest::parseExpectations(istream& _stream) if (it != line.end() && *it == '(') { ++it; - if (it != line.end() && !isdigit(*it)) + if (it != line.end() && !isDigit(*it)) { auto sourceNameStart = it; while (it != line.end() && *it != ':') diff --git a/test/TestCase.cpp b/test/TestCase.cpp index 93b209aba..32de68296 100644 --- a/test/TestCase.cpp +++ b/test/TestCase.cpp @@ -118,7 +118,7 @@ EVMVersionRestrictedTestCase::EVMVersionRestrictedTestCase(string const& _filena string comparator; size_t versionBegin = 0; for (auto character: versionString) - if (!isalpha(character)) + if (!isalpha(character, locale::classic())) { comparator += character; versionBegin++; diff --git a/test/TestCase.h b/test/TestCase.h index 5d4394b11..d63c88b48 100644 --- a/test/TestCase.h +++ b/test/TestCase.h @@ -90,7 +90,7 @@ protected: template static void skipWhitespace(IteratorType& _it, IteratorType _end) { - while (_it != _end && isspace(*_it)) + while (_it != _end && std::isspace(*_it, std::locale::classic())) ++_it; } diff --git a/test/libsolidity/util/BytesUtils.cpp b/test/libsolidity/util/BytesUtils.cpp index f74b256d0..49482a6c6 100644 --- a/test/libsolidity/util/BytesUtils.cpp +++ b/test/libsolidity/util/BytesUtils.cpp @@ -22,6 +22,7 @@ #include #include +#include #include @@ -216,7 +217,7 @@ string BytesUtils::formatString(bytes const& _bytes, size_t _cutOff) os << "\\n"; break; default: - if (isprint(v)) + if (isPrint(static_cast(v))) os << v; else os << "\\x" << toHex(v, HexCase::Lower); diff --git a/test/libsolidity/util/TestFileParser.cpp b/test/libsolidity/util/TestFileParser.cpp index 6652f704a..dbaf52604 100644 --- a/test/libsolidity/util/TestFileParser.cpp +++ b/test/libsolidity/util/TestFileParser.cpp @@ -762,13 +762,16 @@ string TestFileParser::Scanner::scanString() // TODO: use fromHex() from CommonData char TestFileParser::Scanner::scanHexPart() { + auto toLower = [](char _c) -> char { return tolower(_c, locale::classic()); }; + auto isDigit = [](char _c) -> bool { return isdigit(_c, locale::classic()); }; + advance(); // skip 'x' int value{}; - if (isdigit(current())) + if (isDigit(current())) value = current() - '0'; - else if (tolower(current()) >= 'a' && tolower(current()) <= 'f') - value = tolower(current()) - 'a' + 10; + else if (toLower(current()) >= 'a' && toLower(current()) <= 'f') + value = toLower(current()) - 'a' + 10; else BOOST_THROW_EXCEPTION(TestParserError("\\x used with no following hex digits.")); @@ -777,10 +780,10 @@ char TestFileParser::Scanner::scanHexPart() return static_cast(value); value <<= 4; - if (isdigit(current())) + if (isDigit(current())) value |= current() - '0'; - else if (tolower(current()) >= 'a' && tolower(current()) <= 'f') - value |= tolower(current()) - 'a' + 10; + else if (toLower(current()) >= 'a' && toLower(current()) <= 'f') + value |= toLower(current()) - 'a' + 10; advance(); diff --git a/test/tools/ossfuzz/strictasm_diff_ossfuzz.cpp b/test/tools/ossfuzz/strictasm_diff_ossfuzz.cpp index c6d52a62e..dce6c6162 100644 --- a/test/tools/ossfuzz/strictasm_diff_ossfuzz.cpp +++ b/test/tools/ossfuzz/strictasm_diff_ossfuzz.cpp @@ -28,6 +28,7 @@ #include #include +#include #include @@ -53,7 +54,7 @@ extern "C" int LLVMFuzzerTestOneInput(uint8_t const* _data, size_t _size) string input(reinterpret_cast(_data), _size); if (std::any_of(input.begin(), input.end(), [](char c) { - return ((static_cast(c) > 127) || !(std::isprint(c) || (c == '\n') || (c == '\t'))); + return ((static_cast(c) > 127) || !(isPrint(c) || (c == '\n') || (c == '\t'))); })) return 0; diff --git a/test/tools/yulopti.cpp b/test/tools/yulopti.cpp index 5c4f00f0a..ad98ab9be 100644 --- a/test/tools/yulopti.cpp +++ b/test/tools/yulopti.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -155,7 +156,7 @@ public: return ( !boost::algorithm::iequals(get<1>(_a), get<1>(_b)) ? boost::algorithm::lexicographical_compare(get<1>(_a), get<1>(_b), boost::algorithm::is_iless()) : - tolower(get<0>(_a)) < tolower(get<0>(_b)) + toLower(get<0>(_a)) < toLower(get<0>(_b)) ); });