2017-06-26 13:28:24 +00:00
|
|
|
|
/*
|
|
|
|
|
This file is part of solidity.
|
|
|
|
|
|
|
|
|
|
solidity is free software: you can redistribute it and/or modify
|
|
|
|
|
it under the terms of the GNU General Public License as published by
|
|
|
|
|
the Free Software Foundation, either version 3 of the License, or
|
|
|
|
|
(at your option) any later version.
|
|
|
|
|
|
|
|
|
|
solidity is distributed in the hope that it will be useful,
|
|
|
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
|
GNU General Public License for more details.
|
|
|
|
|
|
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
|
|
|
along with solidity. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
|
*/
|
|
|
|
|
/**
|
|
|
|
|
* Unit tests for UTF-8 validation.
|
|
|
|
|
*/
|
|
|
|
|
|
2020-01-06 10:52:23 +00:00
|
|
|
|
#include <libsolutil/CommonData.h>
|
|
|
|
|
#include <libsolutil/UTF8.h>
|
2017-06-26 13:28:24 +00:00
|
|
|
|
|
2020-01-14 16:48:17 +00:00
|
|
|
|
#include <test/Common.h>
|
2017-06-26 13:28:24 +00:00
|
|
|
|
|
2020-01-14 14:55:31 +00:00
|
|
|
|
#include <boost/test/unit_test.hpp>
|
|
|
|
|
|
2017-06-26 13:28:24 +00:00
|
|
|
|
using namespace std;
|
|
|
|
|
|
2019-12-23 15:50:30 +00:00
|
|
|
|
namespace solidity::util::test
|
2017-06-26 13:28:24 +00:00
|
|
|
|
{
|
|
|
|
|
|
2020-07-08 15:56:14 +00:00
|
|
|
|
BOOST_AUTO_TEST_SUITE(UTF8, *boost::unit_test::label("nooptions"))
|
2017-06-26 13:28:24 +00:00
|
|
|
|
|
|
|
|
|
namespace {
|
|
|
|
|
|
|
|
|
|
bool isValidUTF8(string const& _value)
|
|
|
|
|
{
|
|
|
|
|
size_t pos;
|
|
|
|
|
return validateUTF8(asString(fromHex(_value)), pos);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bool isInvalidUTF8(string const& _value, size_t _expectedPos)
|
|
|
|
|
{
|
|
|
|
|
size_t pos;
|
|
|
|
|
if (validateUTF8(asString(fromHex(_value)), pos))
|
|
|
|
|
return false;
|
|
|
|
|
if (pos != _expectedPos)
|
|
|
|
|
return false;
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
BOOST_AUTO_TEST_CASE(valid)
|
|
|
|
|
{
|
|
|
|
|
BOOST_CHECK(isValidUTF8("00"));
|
|
|
|
|
BOOST_CHECK(isValidUTF8("20"));
|
|
|
|
|
BOOST_CHECK(isValidUTF8("7f"));
|
|
|
|
|
BOOST_CHECK(isValidUTF8("c281"));
|
|
|
|
|
BOOST_CHECK(isValidUTF8("df81"));
|
|
|
|
|
BOOST_CHECK(isValidUTF8("e0a081"));
|
|
|
|
|
BOOST_CHECK(isValidUTF8("e18081"));
|
|
|
|
|
BOOST_CHECK(isValidUTF8("ec8081"));
|
|
|
|
|
BOOST_CHECK(isValidUTF8("ed8081"));
|
|
|
|
|
BOOST_CHECK(isValidUTF8("ee8081"));
|
|
|
|
|
BOOST_CHECK(isValidUTF8("ef8081"));
|
|
|
|
|
BOOST_CHECK(isValidUTF8("f0908081"));
|
|
|
|
|
BOOST_CHECK(isValidUTF8("f3808081"));
|
|
|
|
|
BOOST_CHECK(isValidUTF8("f2808081"));
|
|
|
|
|
BOOST_CHECK(isValidUTF8("f3808081"));
|
|
|
|
|
BOOST_CHECK(isValidUTF8("f48e8081"));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
BOOST_AUTO_TEST_CASE(invalid)
|
|
|
|
|
{
|
|
|
|
|
// anything between 0x80 and 0xc0 is disallowed
|
|
|
|
|
BOOST_CHECK(isInvalidUTF8("80", 0)); // invalid per table 3.6
|
|
|
|
|
BOOST_CHECK(isInvalidUTF8("a0", 0)); // invalid per table 3.6
|
|
|
|
|
BOOST_CHECK(isInvalidUTF8("c0", 0)); // invalid per table 3.7
|
|
|
|
|
BOOST_CHECK(isInvalidUTF8("c1", 0)); // invalid per table 3.7
|
|
|
|
|
BOOST_CHECK(isInvalidUTF8("c2", 0)); // too short (position is reported as the first byte)
|
|
|
|
|
BOOST_CHECK(isInvalidUTF8("e08081", 2)); // e0 must be followed by >= a0
|
|
|
|
|
BOOST_CHECK(isInvalidUTF8("e180", 0)); // too short
|
|
|
|
|
BOOST_CHECK(isInvalidUTF8("ec80", 0)); // too short
|
|
|
|
|
BOOST_CHECK(isInvalidUTF8("f08f8001", 2)); // f0 must be followed by >= 90
|
|
|
|
|
BOOST_CHECK(isInvalidUTF8("f18080", 0)); // too short
|
|
|
|
|
BOOST_CHECK(isInvalidUTF8("f4908081", 2)); // f4 must be followed by < 90
|
|
|
|
|
// anything above 0xf7 is disallowed
|
|
|
|
|
BOOST_CHECK(isInvalidUTF8("f8", 0)); // invalid per table 3.7
|
|
|
|
|
BOOST_CHECK(isInvalidUTF8("f9", 0)); // invalid per table 3.7
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
BOOST_AUTO_TEST_CASE(corpus)
|
|
|
|
|
{
|
|
|
|
|
string source = R"(
|
2017-06-26 13:29:00 +00:00
|
|
|
|
κόσμε
|
|
|
|
|
|
|
|
|
|
hélló
|
|
|
|
|
|
|
|
|
|
Ā ā Ă ă Ą ą
|
|
|
|
|
|
|
|
|
|
ƀ Ɓ Ƃ ƃ Ƅ ƅ
|
|
|
|
|
|
|
|
|
|
ɐ ɑ ɒ ɓ ɔ ɕ
|
|
|
|
|
|
|
|
|
|
ʰ ʱ ʲ ʳ ʴ ʵ
|
|
|
|
|
|
|
|
|
|
̀ ́ ̂ ̃ ̄ ̅
|
|
|
|
|
|
|
|
|
|
ϩ Ϫ ϫ Ϭ ϭ Ϯ
|
|
|
|
|
|
|
|
|
|
Ё Ђ Ѓ Є Ѕ І
|
|
|
|
|
|
|
|
|
|
Ա Բ Գ Դ Ե Զ
|
|
|
|
|
|
|
|
|
|
ק ר ש ת װ ױ
|
|
|
|
|
|
|
|
|
|
ځ ڂ ڃ ڄ څ چ
|
|
|
|
|
|
|
|
|
|
ऑ ऒ ओ औ क ख
|
|
|
|
|
|
|
|
|
|
ও ঔ ক খ গ ঘ
|
|
|
|
|
|
|
|
|
|
ਘ ਙ ਚ ਛ ਜ ਝ
|
|
|
|
|
|
|
|
|
|
ઓ ઔ ક ખ ગ ઘ
|
|
|
|
|
|
|
|
|
|
ଗ ଘ ଙ ଚ ଛ ଜ
|
|
|
|
|
|
|
|
|
|
ஔ க ங ச ஜ ஞ
|
|
|
|
|
|
|
|
|
|
ఎ ఏ ఐ ఒ ఓ ఔ
|
|
|
|
|
|
|
|
|
|
ಓ ಔ ಕ ಖ ಗ ಘ
|
|
|
|
|
|
|
|
|
|
ഐ ഒ ഓ ഔ ക
|
|
|
|
|
|
|
|
|
|
ฒ ณ ด ต ถ ท
|
|
|
|
|
|
|
|
|
|
ມ ຢ ຣ ລ ວ ສ
|
|
|
|
|
|
|
|
|
|
༄ ༅ ༆ ༇ ༈ ༉
|
|
|
|
|
|
|
|
|
|
Ⴑ Ⴒ Ⴓ Ⴔ Ⴕ Ⴖ
|
|
|
|
|
|
|
|
|
|
ᄌ ᄍ ᄎ ᄏ ᄐ
|
|
|
|
|
|
|
|
|
|
Ḕ ḕ Ḗ ḗ Ḙ ḙ Ḛ
|
|
|
|
|
|
2018-09-03 13:01:15 +00:00
|
|
|
|
ἐ ἑ ἒ ἓ ἔ ἕ
|
2017-06-26 13:29:00 +00:00
|
|
|
|
|
|
|
|
|
₠ ₡ ₢ ₣ ₤ ₥
|
|
|
|
|
|
|
|
|
|
⃐ ⃑ ⃒ ⃓ ⃔ ⃕ ⃖ ⃗ ⃘ ⃙ ⃚
|
|
|
|
|
|
|
|
|
|
ℋ ℌ ℍ ℎ ℏ ℐ ℑ
|
|
|
|
|
|
|
|
|
|
⅓ ⅔ ⅕ ⅖ ⅗
|
|
|
|
|
|
|
|
|
|
∬ ∭ ∮ ∯ ∰
|
|
|
|
|
|
|
|
|
|
⌖ ⌗ ⌘ ⌙ ⌚ ⌛
|
|
|
|
|
|
|
|
|
|
␀ ␁ ␂ ␃ ␄ ␅
|
|
|
|
|
|
2018-09-03 13:01:15 +00:00
|
|
|
|
⑀ ⑁ ⑂ ⑃ ⑄
|
2017-06-26 13:29:00 +00:00
|
|
|
|
|
2018-09-03 13:01:15 +00:00
|
|
|
|
① ② ③ ④ ⑤
|
2017-06-26 13:29:00 +00:00
|
|
|
|
|
|
|
|
|
╘ ╙ ╚ ╛ ╜ ╝
|
|
|
|
|
|
|
|
|
|
▁ ▂ ▃ ▄ ▅ ▆
|
|
|
|
|
|
|
|
|
|
▤ ▥ ▦ ▧ ▨
|
|
|
|
|
|
|
|
|
|
♔ ♕ ♖ ♗ ♘ ♙
|
|
|
|
|
|
|
|
|
|
✈ ✉ ✌ ✍ ✎
|
|
|
|
|
|
|
|
|
|
ぁ あ ぃ い ぅ
|
|
|
|
|
|
|
|
|
|
ァ ア ィ イ ゥ
|
|
|
|
|
|
|
|
|
|
ㄅ ㄆ ㄇ ㄈ ㄉ
|
|
|
|
|
|
|
|
|
|
ㄱ ㄲ ㄳ ㄴ ㄵ
|
|
|
|
|
|
|
|
|
|
㆚ ㆛ ㆜ ㆝ ㆞
|
|
|
|
|
|
|
|
|
|
㈀ ㈁ ㈂ ㈃ ㈄
|
|
|
|
|
|
|
|
|
|
㌀ ㌁ ㌂ ㌃ ㌄
|
|
|
|
|
|
|
|
|
|
乺 乻 乼 乽 乾
|
|
|
|
|
|
|
|
|
|
걺 걻 걼 걽 걾
|
|
|
|
|
|
|
|
|
|
豈 更 車 賈 滑
|
|
|
|
|
|
|
|
|
|
שּׁ שּׂ אַ אָ אּ
|
|
|
|
|
|
2018-09-03 13:01:15 +00:00
|
|
|
|
ﮄ ﮅ ﮆ ﮇ ﮈ ﮉ
|
2017-06-26 13:29:00 +00:00
|
|
|
|
|
|
|
|
|
ﺵ ﺶ ﺷ ﺸ
|
|
|
|
|
|
|
|
|
|
「 」 、 ・ ヲ ァ ィ ゥ
|
2017-06-26 13:28:24 +00:00
|
|
|
|
)";
|
2017-06-26 13:29:00 +00:00
|
|
|
|
size_t pos;
|
|
|
|
|
BOOST_CHECK(validateUTF8(source, pos));
|
2017-06-26 13:28:24 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
BOOST_AUTO_TEST_SUITE_END()
|
|
|
|
|
|
|
|
|
|
}
|