diff --git a/native/jni/Android.mk b/native/jni/Android.mk index 4786ef6c7..ecc0e4c96 100644 --- a/native/jni/Android.mk +++ b/native/jni/Android.mk @@ -71,6 +71,7 @@ LATIN_IME_CORE_SRC_FILES := \ suggest/core/policy/weighting.cpp \ suggest/core/session/dic_traverse_session.cpp \ $(addprefix suggest/policyimpl/dictionary/, \ + bigram/bigram_list_reading_utils.cpp \ dictionary_structure_with_buffer_policy_factory.cpp \ dynamic_patricia_trie_node_reader.cpp \ dynamic_patricia_trie_policy.cpp \ diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.cpp b/native/jni/src/suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.cpp index 9e7d7a391..cb73a577e 100644 --- a/native/jni/src/suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.cpp +++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.cpp @@ -36,27 +36,4 @@ const int TaUtils::SHORTCUT_LIST_SIZE_FIELD_SIZE = 2; // The numeric value of the shortcut probability that means 'whitelist'. const int TaUtils::WHITELIST_SHORTCUT_PROBABILITY = 15; -/* static */ int TaUtils::getBigramAddressAndForwardPointer( - const uint8_t *const dictRoot, const TerminalAttributeFlags flags, - int *const pos) { - int offset = 0; - const int origin = *pos; - switch (MASK_ATTRIBUTE_ADDRESS_TYPE & flags) { - case FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: - offset = ByteArrayUtils::readUint8AndAdvancePosition(dictRoot, pos); - break; - case FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: - offset = ByteArrayUtils::readUint16AndAdvancePosition(dictRoot, pos); - break; - case FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES: - offset = ByteArrayUtils::readUint24AndAdvancePosition(dictRoot, pos); - break; - } - if (isOffsetNegative(flags)) { - return origin - offset; - } else { - return origin + offset; - } -} - } // namespace latinime diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.h b/native/jni/src/suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.h index e1e3e1618..c28f43ff7 100644 --- a/native/jni/src/suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.h +++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.h @@ -28,7 +28,6 @@ namespace latinime { class BinaryDictionaryTerminalAttributesReadingUtils { public: typedef uint8_t TerminalAttributeFlags; - typedef TerminalAttributeFlags BigramFlags; typedef TerminalAttributeFlags ShortcutFlags; static AK_FORCE_INLINE TerminalAttributeFlags getFlagsAndForwardPointer( @@ -44,20 +43,6 @@ class BinaryDictionaryTerminalAttributesReadingUtils { return (flags & FLAG_ATTRIBUTE_HAS_NEXT) != 0; } - // Bigrams reading methods - static AK_FORCE_INLINE void skipExistingBigrams( - const uint8_t *const dictRoot, int *const pos) { - BigramFlags flags = getFlagsAndForwardPointer(dictRoot, pos); - while (hasNext(flags)) { - *pos += attributeAddressSize(flags); - flags = getFlagsAndForwardPointer(dictRoot, pos); - } - *pos += attributeAddressSize(flags); - } - - static int getBigramAddressAndForwardPointer( - const uint8_t *const dictRoot, const BigramFlags flags, int *const pos); - // Shortcuts reading methods // This method returns the size of the shortcut list region excluding the shortcut list size // field at the beginning. diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigrams/bigram_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_policy.h similarity index 64% rename from native/jni/src/suggest/policyimpl/dictionary/bigrams/bigram_list_policy.h rename to native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_policy.h index a9e5da3c7..beb9bee27 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigrams/bigram_list_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_policy.h @@ -21,8 +21,7 @@ #include "defines.h" #include "suggest/core/policy/dictionary_bigrams_structure_policy.h" -// TODO: Move bigrams reading methods to policyimpl. -#include "suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.h" +#include "suggest/policyimpl/dictionary/bigram/bigram_list_reading_utils.h" namespace latinime { @@ -34,19 +33,16 @@ class BigramListPolicy : public DictionaryBigramsStructurePolicy { void getNextBigram(int *const outBigramPos, int *const outProbability, bool *const outHasNext, int *const pos) const { - const BinaryDictionaryTerminalAttributesReadingUtils::BigramFlags flags = - BinaryDictionaryTerminalAttributesReadingUtils::getFlagsAndForwardPointer( - mBigramsBuf, pos); - *outBigramPos = - BinaryDictionaryTerminalAttributesReadingUtils::getBigramAddressAndForwardPointer( + const BigramListReadingUtils::BigramFlags flags = + BigramListReadingUtils::getFlagsAndForwardPointer(mBigramsBuf, pos); + *outBigramPos = BigramListReadingUtils::getBigramAddressAndForwardPointer( mBigramsBuf, flags, pos); - *outProbability = - BinaryDictionaryTerminalAttributesReadingUtils::getProbabilityFromFlags(flags); - *outHasNext = BinaryDictionaryTerminalAttributesReadingUtils::hasNext(flags); + *outProbability = BigramListReadingUtils::getProbabilityFromFlags(flags); + *outHasNext = BigramListReadingUtils::hasNext(flags); } void skipAllBigrams(int *const pos) const { - BinaryDictionaryTerminalAttributesReadingUtils::skipExistingBigrams(mBigramsBuf, pos); + BigramListReadingUtils::skipExistingBigrams(mBigramsBuf, pos); } private: diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_reading_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_reading_utils.cpp new file mode 100644 index 000000000..6f4fcbfc2 --- /dev/null +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_reading_utils.cpp @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "suggest/policyimpl/dictionary/bigram/bigram_list_reading_utils.h" + +#include "suggest/core/dictionary/byte_array_utils.h" + +namespace latinime { + +const BigramListReadingUtils::BigramFlags BigramListReadingUtils::MASK_ATTRIBUTE_ADDRESS_TYPE = + 0x30; +const BigramListReadingUtils::BigramFlags + BigramListReadingUtils::FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE = 0x10; +const BigramListReadingUtils::BigramFlags + BigramListReadingUtils::FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES = 0x20; +const BigramListReadingUtils::BigramFlags + BigramListReadingUtils::FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES = 0x30; +const BigramListReadingUtils::BigramFlags + BigramListReadingUtils::FLAG_ATTRIBUTE_OFFSET_NEGATIVE = 0x40; +// Flag for presence of more attributes +const BigramListReadingUtils::BigramFlags BigramListReadingUtils::FLAG_ATTRIBUTE_HAS_NEXT = 0x80; +// Mask for attribute probability, stored on 4 bits inside the flags byte. +const BigramListReadingUtils::BigramFlags + BigramListReadingUtils::MASK_ATTRIBUTE_PROBABILITY = 0x0F; +const int BigramListReadingUtils::ATTRIBUTE_ADDRESS_SHIFT = 4; + +/* static */ int BigramListReadingUtils::getBigramAddressAndForwardPointer( + const uint8_t *const bigramsBuf, const BigramFlags flags, int *const pos) { + int offset = 0; + const int origin = *pos; + switch (MASK_ATTRIBUTE_ADDRESS_TYPE & flags) { + case FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: + offset = ByteArrayUtils::readUint8AndAdvancePosition(bigramsBuf, pos); + break; + case FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: + offset = ByteArrayUtils::readUint16AndAdvancePosition(bigramsBuf, pos); + break; + case FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES: + offset = ByteArrayUtils::readUint24AndAdvancePosition(bigramsBuf, pos); + break; + } + if (isOffsetNegative(flags)) { + return origin - offset; + } else { + return origin + offset; + } +} + +} // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_reading_utils.h b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_reading_utils.h new file mode 100644 index 000000000..6b2bfe8c9 --- /dev/null +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_reading_utils.h @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_BIGRAM_LIST_READING_UTILS_H +#define LATINIME_BIGRAM_LIST_READING_UTILS_H + +#include + +#include "defines.h" +#include "suggest/core/dictionary/byte_array_utils.h" + +namespace latinime { + +class BigramListReadingUtils { +public: + typedef uint8_t BigramFlags; + + static AK_FORCE_INLINE BigramFlags getFlagsAndForwardPointer( + const uint8_t *const bigramsBuf, int *const pos) { + return ByteArrayUtils::readUint8AndAdvancePosition(bigramsBuf, pos); + } + + static AK_FORCE_INLINE int getProbabilityFromFlags(const BigramFlags flags) { + return flags & MASK_ATTRIBUTE_PROBABILITY; + } + + static AK_FORCE_INLINE bool hasNext(const BigramFlags flags) { + return (flags & FLAG_ATTRIBUTE_HAS_NEXT) != 0; + } + + // Bigrams reading methods + static AK_FORCE_INLINE void skipExistingBigrams(const uint8_t *const bigramsBuf, + int *const pos) { + BigramFlags flags = getFlagsAndForwardPointer(bigramsBuf, pos); + while (hasNext(flags)) { + *pos += attributeAddressSize(flags); + flags = getFlagsAndForwardPointer(bigramsBuf, pos); + } + *pos += attributeAddressSize(flags); + } + + static int getBigramAddressAndForwardPointer(const uint8_t *const bigramsBuf, + const BigramFlags flags, int *const pos); + +private: + DISALLOW_IMPLICIT_CONSTRUCTORS(BigramListReadingUtils); + + static const BigramFlags MASK_ATTRIBUTE_ADDRESS_TYPE; + static const BigramFlags FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE; + static const BigramFlags FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES; + static const BigramFlags FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES; + static const BigramFlags FLAG_ATTRIBUTE_OFFSET_NEGATIVE; + static const BigramFlags FLAG_ATTRIBUTE_HAS_NEXT; + static const BigramFlags MASK_ATTRIBUTE_PROBABILITY; + static const int ATTRIBUTE_ADDRESS_SHIFT; + + static AK_FORCE_INLINE bool isOffsetNegative(const BigramFlags flags) { + return (flags & FLAG_ATTRIBUTE_OFFSET_NEGATIVE) != 0; + } + + static AK_FORCE_INLINE int attributeAddressSize(const BigramFlags flags) { + return (flags & MASK_ATTRIBUTE_ADDRESS_TYPE) >> ATTRIBUTE_ADDRESS_SHIFT; + /* Note: this is a value-dependant optimization of what may probably be + more readably written this way: + switch (flags * BinaryFormat::MASK_ATTRIBUTE_ADDRESS_TYPE) { + case FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: return 1; + case FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: return 2; + case FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTE: return 3; + default: return 0; + } + */ + } +}; +} // namespace latinime +#endif // LATINIME_BIGRAM_LIST_READING_UTILS_H diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp index e8799e574..a6214d3e4 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp @@ -18,7 +18,7 @@ #include "suggest/core/dictionary/binary_dictionary_info.h" #include "suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.h" -#include "suggest/policyimpl/dictionary/bigrams/bigram_list_policy.h" +#include "suggest/core/policy/dictionary_bigrams_structure_policy.h" #include "suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h" namespace latinime { diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h index 641ac9b5b..4f22db738 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h @@ -20,7 +20,6 @@ #include #include "defines.h" -#include "suggest/policyimpl/dictionary/bigrams/bigram_list_policy.h" #include "suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/patricia_trie_reading_utils.h" diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h index 2386d937f..1e94140c4 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h @@ -21,7 +21,7 @@ #include "defines.h" #include "suggest/core/policy/dictionary_structure_with_buffer_policy.h" -#include "suggest/policyimpl/dictionary/bigrams/bigram_list_policy.h" +#include "suggest/policyimpl/dictionary/bigram/bigram_list_policy.h" namespace latinime { diff --git a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h index bebee39fb..2f81aedb8 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h @@ -21,7 +21,7 @@ #include "defines.h" #include "suggest/core/policy/dictionary_structure_with_buffer_policy.h" -#include "suggest/policyimpl/dictionary/bigrams/bigram_list_policy.h" +#include "suggest/policyimpl/dictionary/bigram/bigram_list_policy.h" namespace latinime {