From 29dcf97e7472bde94d591c743c5311b641fd8238 Mon Sep 17 00:00:00 2001 From: Keisuke Kuroyanagi Date: Wed, 3 Sep 2014 13:57:32 +0900 Subject: [PATCH] Add CharUtilsTest. Change-Id: I69b57d88a5376393b5d09d8b8e056c82cb8b8a3c --- native/jni/NativeFileList.mk | 1 + native/jni/src/utils/char_utils.cpp | 4 +- native/jni/src/utils/char_utils.h | 26 ++--- native/jni/tests/utils/char_utils_test.cpp | 122 +++++++++++++++++++++ 4 files changed, 138 insertions(+), 15 deletions(-) create mode 100644 native/jni/tests/utils/char_utils_test.cpp diff --git a/native/jni/NativeFileList.mk b/native/jni/NativeFileList.mk index f21b54c9b..4f77388fd 100644 --- a/native/jni/NativeFileList.mk +++ b/native/jni/NativeFileList.mk @@ -134,5 +134,6 @@ LATIN_IME_CORE_TEST_FILES := \ suggest/policyimpl/dictionary/utils/trie_map_test.cpp \ suggest/policyimpl/utils/damerau_levenshtein_edit_distance_policy_test.cpp \ utils/autocorrection_threshold_utils_test.cpp \ + utils/char_utils_test.cpp \ utils/int_array_view_test.cpp \ utils/time_keeper_test.cpp diff --git a/native/jni/src/utils/char_utils.cpp b/native/jni/src/utils/char_utils.cpp index b17e0847d..3bb9055b2 100644 --- a/native/jni/src/utils/char_utils.cpp +++ b/native/jni/src/utils/char_utils.cpp @@ -1057,11 +1057,11 @@ static int compare_pair_capital(const void *a, const void *b) { - static_cast((static_cast(b))->capital); } -/* static */ unsigned short CharUtils::latin_tolower(const unsigned short c) { +/* static */ int CharUtils::latin_tolower(const int c) { struct LatinCapitalSmallPair *p = static_cast(bsearch(&c, SORTED_CHAR_MAP, NELEMS(SORTED_CHAR_MAP), sizeof(SORTED_CHAR_MAP[0]), compare_pair_capital)); - return p ? p->small : c; + return p ? static_cast(p->small) : c; } /* diff --git a/native/jni/src/utils/char_utils.h b/native/jni/src/utils/char_utils.h index 63786502b..5e9cdd9b2 100644 --- a/native/jni/src/utils/char_utils.h +++ b/native/jni/src/utils/char_utils.h @@ -27,20 +27,14 @@ namespace latinime { class CharUtils { public: + static const std::vector EMPTY_STRING; + static AK_FORCE_INLINE bool isAsciiUpper(int c) { // Note: isupper(...) reports false positives for some Cyrillic characters, causing them to // be incorrectly lower-cased using toAsciiLower(...) rather than latin_tolower(...). return (c >= 'A' && c <= 'Z'); } - static AK_FORCE_INLINE int toAsciiLower(int c) { - return c - 'A' + 'a'; - } - - static AK_FORCE_INLINE bool isAscii(int c) { - return isascii(c) != 0; - } - static AK_FORCE_INLINE int toLowerCase(const int c) { if (isAsciiUpper(c)) { return toAsciiLower(c); @@ -48,7 +42,7 @@ class CharUtils { if (isAscii(c)) { return c; } - return static_cast(latin_tolower(static_cast(c))); + return latin_tolower(c); } static AK_FORCE_INLINE int toBaseLowerCase(const int c) { @@ -59,7 +53,6 @@ class CharUtils { // TODO: Do not hardcode here return codePoint == KEYCODE_SINGLE_QUOTE || codePoint == KEYCODE_HYPHEN_MINUS; } - static AK_FORCE_INLINE int getCodePointCount(const int arraySize, const int *const codePoints) { int size = 0; for (; size < arraySize; ++size) { @@ -91,9 +84,6 @@ class CharUtils { return codePoint >= MIN_UNICODE_CODE_POINT && codePoint <= MAX_UNICODE_CODE_POINT; } - static unsigned short latin_tolower(const unsigned short c); - static const std::vector EMPTY_STRING; - // Returns updated code point count. Returns 0 when the code points cannot be marked as a // Beginning-of-Sentence. static AK_FORCE_INLINE int attachBeginningOfSentenceMarker(int *const codePoints, @@ -125,6 +115,16 @@ class CharUtils { */ static const int BASE_CHARS_SIZE = 0x0500; static const unsigned short BASE_CHARS[BASE_CHARS_SIZE]; + + static AK_FORCE_INLINE bool isAscii(int c) { + return isascii(c) != 0; + } + + static AK_FORCE_INLINE int toAsciiLower(int c) { + return c - 'A' + 'a'; + } + + static int latin_tolower(const int c); }; } // namespace latinime #endif // LATINIME_CHAR_UTILS_H diff --git a/native/jni/tests/utils/char_utils_test.cpp b/native/jni/tests/utils/char_utils_test.cpp new file mode 100644 index 000000000..01d534043 --- /dev/null +++ b/native/jni/tests/utils/char_utils_test.cpp @@ -0,0 +1,122 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "utils/char_utils.h" + +#include + +#include "defines.h" + +namespace latinime { +namespace { + +TEST(CharUtilsTest, TestIsAsciiUpper) { + EXPECT_TRUE(CharUtils::isAsciiUpper('A')); + EXPECT_TRUE(CharUtils::isAsciiUpper('Z')); + EXPECT_FALSE(CharUtils::isAsciiUpper('a')); + EXPECT_FALSE(CharUtils::isAsciiUpper('z')); + EXPECT_FALSE(CharUtils::isAsciiUpper('@')); + EXPECT_FALSE(CharUtils::isAsciiUpper(' ')); + EXPECT_FALSE(CharUtils::isAsciiUpper(0x00C0 /* LATIN CAPITAL LETTER A WITH GRAVE */)); + EXPECT_FALSE(CharUtils::isAsciiUpper(0x00E0 /* LATIN SMALL LETTER A WITH GRAVE */)); + EXPECT_FALSE(CharUtils::isAsciiUpper(0x03C2 /* GREEK SMALL LETTER FINAL SIGMA */)); + EXPECT_FALSE(CharUtils::isAsciiUpper(0x0410 /* CYRILLIC CAPITAL LETTER A */)); + EXPECT_FALSE(CharUtils::isAsciiUpper(0x0430 /* CYRILLIC SMALL LETTER A */)); + EXPECT_FALSE(CharUtils::isAsciiUpper(0x3042 /* HIRAGANA LETTER A */)); + EXPECT_FALSE(CharUtils::isAsciiUpper(0x1F36A /* COOKIE */)); +} + +TEST(CharUtilsTest, TestToLowerCase) { + EXPECT_EQ('a', CharUtils::toLowerCase('A')); + EXPECT_EQ('z', CharUtils::toLowerCase('Z')); + EXPECT_EQ('a', CharUtils::toLowerCase('a')); + EXPECT_EQ('z', CharUtils::toLowerCase('z')); + EXPECT_EQ('@', CharUtils::toLowerCase('@')); + EXPECT_EQ(' ', CharUtils::toLowerCase(' ')); + EXPECT_EQ(0x00E0 /* LATIN SMALL LETTER A WITH GRAVE */, + CharUtils::toLowerCase(0x00C0 /* LATIN CAPITAL LETTER A WITH GRAVE */)); + EXPECT_EQ(0x00E0 /* LATIN SMALL LETTER A WITH GRAVE */, + CharUtils::toLowerCase(0x00E0 /* LATIN SMALL LETTER A WITH GRAVE */)); + EXPECT_EQ(0x03C2 /* GREEK SMALL LETTER FINAL SIGMA */, + CharUtils::toLowerCase(0x03C2 /* GREEK SMALL LETTER FINAL SIGMA */)); + EXPECT_EQ(0x0430 /* CYRILLIC SMALL LETTER A */, + CharUtils::toLowerCase(0x0410 /* CYRILLIC CAPITAL LETTER A */)); + EXPECT_EQ(0x0430 /* CYRILLIC SMALL LETTER A */, + CharUtils::toLowerCase(0x0430 /* CYRILLIC SMALL LETTER A */)); + EXPECT_EQ(0x3042 /* HIRAGANA LETTER A */, + CharUtils::toLowerCase(0x3042 /* HIRAGANA LETTER A */)); + EXPECT_EQ(0x1F36A /* COOKIE */, CharUtils::toLowerCase(0x1F36A /* COOKIE */)); +} + +TEST(CharUtilsTest, TestToBaseLowerCase) { + EXPECT_EQ('a', CharUtils::toBaseLowerCase('A')); + EXPECT_EQ('z', CharUtils::toBaseLowerCase('Z')); + EXPECT_EQ('a', CharUtils::toBaseLowerCase('a')); + EXPECT_EQ('z', CharUtils::toBaseLowerCase('z')); + EXPECT_EQ('@', CharUtils::toBaseLowerCase('@')); + EXPECT_EQ(' ', CharUtils::toBaseLowerCase(' ')); + EXPECT_EQ('a', CharUtils::toBaseLowerCase(0x00C0 /* LATIN CAPITAL LETTER A WITH GRAVE */)); + EXPECT_EQ('a', CharUtils::toBaseLowerCase(0x00E0 /* LATIN SMALL LETTER A WITH GRAVE */)); + EXPECT_EQ(0x03C2 /* GREEK SMALL LETTER FINAL SIGMA */, + CharUtils::toBaseLowerCase(0x03C2 /* GREEK SMALL LETTER FINAL SIGMA */)); + EXPECT_EQ(0x0430 /* CYRILLIC SMALL LETTER A */, + CharUtils::toBaseLowerCase(0x0410 /* CYRILLIC CAPITAL LETTER A */)); + EXPECT_EQ(0x0430 /* CYRILLIC SMALL LETTER A */, + CharUtils::toBaseLowerCase(0x0430 /* CYRILLIC SMALL LETTER A */)); + EXPECT_EQ(0x3042 /* HIRAGANA LETTER A */, + CharUtils::toBaseLowerCase(0x3042 /* HIRAGANA LETTER A */)); + EXPECT_EQ(0x1F36A /* COOKIE */, CharUtils::toBaseLowerCase(0x1F36A /* COOKIE */)); +} + +TEST(CharUtilsTest, TestToBaseCodePoint) { + EXPECT_EQ('A', CharUtils::toBaseCodePoint('A')); + EXPECT_EQ('Z', CharUtils::toBaseCodePoint('Z')); + EXPECT_EQ('a', CharUtils::toBaseCodePoint('a')); + EXPECT_EQ('z', CharUtils::toBaseCodePoint('z')); + EXPECT_EQ('@', CharUtils::toBaseCodePoint('@')); + EXPECT_EQ(' ', CharUtils::toBaseCodePoint(' ')); + EXPECT_EQ('A', CharUtils::toBaseCodePoint(0x00C0 /* LATIN CAPITAL LETTER A WITH GRAVE */)); + EXPECT_EQ('a', CharUtils::toBaseCodePoint(0x00E0 /* LATIN SMALL LETTER A WITH GRAVE */)); + EXPECT_EQ(0x03C2 /* GREEK SMALL LETTER FINAL SIGMA */, + CharUtils::toBaseLowerCase(0x03C2 /* GREEK SMALL LETTER FINAL SIGMA */)); + EXPECT_EQ(0x0410 /* CYRILLIC CAPITAL LETTER A */, + CharUtils::toBaseCodePoint(0x0410 /* CYRILLIC CAPITAL LETTER A */)); + EXPECT_EQ(0x0430 /* CYRILLIC SMALL LETTER A */, + CharUtils::toBaseCodePoint(0x0430 /* CYRILLIC SMALL LETTER A */)); + EXPECT_EQ(0x3042 /* HIRAGANA LETTER A */, + CharUtils::toBaseCodePoint(0x3042 /* HIRAGANA LETTER A */)); + EXPECT_EQ(0x1F36A /* COOKIE */, CharUtils::toBaseCodePoint(0x1F36A /* COOKIE */)); +} + +TEST(CharUtilsTest, TestIsIntentionalOmissionCodePoint) { + EXPECT_TRUE(CharUtils::isIntentionalOmissionCodePoint('\'')); + EXPECT_TRUE(CharUtils::isIntentionalOmissionCodePoint('-')); + EXPECT_FALSE(CharUtils::isIntentionalOmissionCodePoint('a')); + EXPECT_FALSE(CharUtils::isIntentionalOmissionCodePoint('?')); + EXPECT_FALSE(CharUtils::isIntentionalOmissionCodePoint('/')); +} + +TEST(CharUtilsTest, TestIsInUnicodeSpace) { + EXPECT_FALSE(CharUtils::isInUnicodeSpace(NOT_A_CODE_POINT)); + EXPECT_FALSE(CharUtils::isInUnicodeSpace(CODE_POINT_BEGINNING_OF_SENTENCE)); + EXPECT_TRUE(CharUtils::isInUnicodeSpace('a')); + EXPECT_TRUE(CharUtils::isInUnicodeSpace(0x0410 /* CYRILLIC CAPITAL LETTER A */)); + EXPECT_TRUE(CharUtils::isInUnicodeSpace(0x3042 /* HIRAGANA LETTER A */)); + EXPECT_TRUE(CharUtils::isInUnicodeSpace(0x1F36A /* COOKIE */)); +} + +} // namespace +} // namespace latinime