nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

am 6ae4d79d: Use IntArrayView for dictionary structure policy. 

* commit '6ae4d79d81aa7aea5529d95bb3eb960a273ef411':
  Use IntArrayView for dictionary structure policy.
main
Keisuke Kuroyanagi 2014-08-27 12:31:05 +00:00 committed by Android Git Automerger
parent 6302f53797 6ae4d79d81
commit d18753b151
11 changed files with 117 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
View File

@ -32,6 +32,7 @@
#include "suggest/core/suggest_options.h" #include "suggest/core/suggest_options.h"
#include "suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h" #include "suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h"
#include "utils/char_utils.h" #include "utils/char_utils.h"
#include "utils/int_array_view.h"
#include "utils/jni_data_utils.h" #include "utils/jni_data_utils.h"
#include "utils/log_utils.h" #include "utils/log_utils.h"
#include "utils/time_keeper.h" #include "utils/time_keeper.h"
@ -581,8 +582,9 @@ static bool latinime_BinaryDictionary_migrateNative(JNIEnv *env, jclass clazz, j
return false; return false;
} }
} }
if (!dictionaryStructureWithBufferPolicy->addUnigramEntry(wordCodePoints, if (!dictionaryStructureWithBufferPolicy->addUnigramEntry(
wordCodePointCount, wordProperty.getUnigramProperty())) { CodePointArrayView(wordCodePoints, wordCodePointCount),
wordProperty.getUnigramProperty())) {
LogUtils::logToJava(env, "Cannot add unigram to the new dict."); LogUtils::logToJava(env, "Cannot add unigram to the new dict.");
return false; return false;
} }

16
native/jni/src/suggest/core/dictionary/dictionary.cpp
View File

@ -28,6 +28,7 @@
#include "suggest/core/suggest_options.h" #include "suggest/core/suggest_options.h"
#include "suggest/policyimpl/gesture/gesture_suggest_policy_factory.h" #include "suggest/policyimpl/gesture/gesture_suggest_policy_factory.h"
#include "suggest/policyimpl/typing/typing_suggest_policy_factory.h" #include "suggest/policyimpl/typing/typing_suggest_policy_factory.h"
#include "utils/int_array_view.h"
#include "utils/log_utils.h" #include "utils/log_utils.h"
#include "utils/time_keeper.h" #include "utils/time_keeper.h"
@ -112,8 +113,8 @@ int Dictionary::getMaxProbabilityOfExactMatches(const int *word, int length) con
int Dictionary::getNgramProbability(const PrevWordsInfo *const prevWordsInfo, const int *word, int Dictionary::getNgramProbability(const PrevWordsInfo *const prevWordsInfo, const int *word,
int length) const { int length) const {
TimeKeeper::setCurrentTime(); TimeKeeper::setCurrentTime();
int nextWordPos = mDictionaryStructureWithBufferPolicy->getTerminalPtNodePositionOfWord(word, int nextWordPos = mDictionaryStructureWithBufferPolicy->getTerminalPtNodePositionOfWord(
length, false /* forceLowerCaseSearch */); CodePointArrayView(word, length), false /* forceLowerCaseSearch */);
if (NOT_A_DICT_POS == nextWordPos) return NOT_A_PROBABILITY; if (NOT_A_DICT_POS == nextWordPos) return NOT_A_PROBABILITY;
if (!prevWordsInfo) { if (!prevWordsInfo) {
return getDictionaryStructurePolicy()->getProbabilityOfPtNode( return getDictionaryStructurePolicy()->getProbabilityOfPtNode(
@ -135,12 +136,14 @@ bool Dictionary::addUnigramEntry(const int *const word, const int length,
return false; return false;
} }
TimeKeeper::setCurrentTime(); TimeKeeper::setCurrentTime();
return mDictionaryStructureWithBufferPolicy->addUnigramEntry(word, length, unigramProperty); return mDictionaryStructureWithBufferPolicy->addUnigramEntry(CodePointArrayView(word, length),
unigramProperty);
} }
bool Dictionary::removeUnigramEntry(const int *const codePoints, const int codePointCount) { bool Dictionary::removeUnigramEntry(const int *const codePoints, const int codePointCount) {
TimeKeeper::setCurrentTime(); TimeKeeper::setCurrentTime();
return mDictionaryStructureWithBufferPolicy->removeUnigramEntry(codePoints, codePointCount); return mDictionaryStructureWithBufferPolicy->removeUnigramEntry(
CodePointArrayView(codePoints, codePointCount));
} }
bool Dictionary::addNgramEntry(const PrevWordsInfo *const prevWordsInfo, bool Dictionary::addNgramEntry(const PrevWordsInfo *const prevWordsInfo,
@ -152,7 +155,8 @@ bool Dictionary::addNgramEntry(const PrevWordsInfo *const prevWordsInfo,
bool Dictionary::removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, bool Dictionary::removeNgramEntry(const PrevWordsInfo *const prevWordsInfo,
const int *const word, const int length) { const int *const word, const int length) {
TimeKeeper::setCurrentTime(); TimeKeeper::setCurrentTime();
return mDictionaryStructureWithBufferPolicy->removeNgramEntry(prevWordsInfo, word, length); return mDictionaryStructureWithBufferPolicy->removeNgramEntry(prevWordsInfo,
CodePointArrayView(word, length));
} }
bool Dictionary::flush(const char *const filePath) { bool Dictionary::flush(const char *const filePath) {
@ -181,7 +185,7 @@ const WordProperty Dictionary::getWordProperty(const int *const codePoints,
const int codePointCount) { const int codePointCount) {
TimeKeeper::setCurrentTime(); TimeKeeper::setCurrentTime();
return mDictionaryStructureWithBufferPolicy->getWordProperty( return mDictionaryStructureWithBufferPolicy->getWordProperty(
codePoints, codePointCount); CodePointArrayView(codePoints, codePointCount));
} }
int Dictionary::getNextWordAndNextToken(const int token, int *const outCodePoints, int Dictionary::getNextWordAndNextToken(const int token, int *const outCodePoints,

23
native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h
View File

@ -21,6 +21,7 @@
#include "defines.h" #include "defines.h"
#include "suggest/core/dictionary/property/word_property.h" #include "suggest/core/dictionary/property/word_property.h"
#include "utils/int_array_view.h"
namespace latinime { namespace latinime {
@ -49,33 +50,32 @@ class DictionaryStructureWithBufferPolicy {
DicNodeVector *const childDicNodes) const = 0; DicNodeVector *const childDicNodes) const = 0;
virtual int getCodePointsAndProbabilityAndReturnCodePointCount( virtual int getCodePointsAndProbabilityAndReturnCodePointCount(
const int nodePos, const int maxCodePointCount, int *const outCodePoints, const int ptNodePos, const int maxCodePointCount, int *const outCodePoints,
int *const outUnigramProbability) const = 0; int *const outUnigramProbability) const = 0;
virtual int getTerminalPtNodePositionOfWord(const int *const inWord, virtual int getTerminalPtNodePositionOfWord(const CodePointArrayView wordCodePoints,
const int length, const bool forceLowerCaseSearch) const = 0; const bool forceLowerCaseSearch) const = 0;
virtual int getProbability(const int unigramProbability, virtual int getProbability(const int unigramProbability, const int bigramProbability) const = 0;
const int bigramProbability) const = 0;
virtual int getProbabilityOfPtNode(const int *const prevWordsPtNodePos, virtual int getProbabilityOfPtNode(const int *const prevWordsPtNodePos,
const int nodePos) const = 0; const int ptNodePos) const = 0;
virtual void iterateNgramEntries(const int *const prevWordsPtNodePos, virtual void iterateNgramEntries(const int *const prevWordsPtNodePos,
NgramListener *const listener) const = 0; NgramListener *const listener) const = 0;
virtual int getShortcutPositionOfPtNode(const int nodePos) const = 0; virtual int getShortcutPositionOfPtNode(const int ptNodePos) const = 0;
virtual const DictionaryHeaderStructurePolicy *getHeaderStructurePolicy() const = 0; virtual const DictionaryHeaderStructurePolicy *getHeaderStructurePolicy() const = 0;
virtual const DictionaryShortcutsStructurePolicy *getShortcutsStructurePolicy() const = 0; virtual const DictionaryShortcutsStructurePolicy *getShortcutsStructurePolicy() const = 0;
// Returns whether the update was success or not. // Returns whether the update was success or not.
virtual bool addUnigramEntry(const int *const word, const int length, virtual bool addUnigramEntry(const CodePointArrayView wordCodePoints,
const UnigramProperty *const unigramProperty) = 0; const UnigramProperty *const unigramProperty) = 0;
// Returns whether the update was success or not. // Returns whether the update was success or not.
virtual bool removeUnigramEntry(const int *const word, const int length) = 0; virtual bool removeUnigramEntry(const CodePointArrayView wordCodePoints) = 0;
// Returns whether the update was success or not. // Returns whether the update was success or not.
virtual bool addNgramEntry(const PrevWordsInfo *const prevWordsInfo, virtual bool addNgramEntry(const PrevWordsInfo *const prevWordsInfo,
@ -83,7 +83,7 @@ class DictionaryStructureWithBufferPolicy {
// Returns whether the update was success or not. // Returns whether the update was success or not.
virtual bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, virtual bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo,
const int *const word, const int length) = 0; const CodePointArrayView wordCodePoints) = 0;
// Returns whether the flush was success or not. // Returns whether the flush was success or not.
virtual bool flush(const char *const filePath) = 0; virtual bool flush(const char *const filePath) = 0;
@ -99,8 +99,7 @@ class DictionaryStructureWithBufferPolicy {
const int maxResultLength) = 0; const int maxResultLength) = 0;
// Used for testing. // Used for testing.
virtual const WordProperty getWordProperty(const int *const codePonts, virtual const WordProperty getWordProperty(const CodePointArrayView wordCodePoints) const = 0;
const int codePointCount) const = 0;
// Method to iterate all words in the dictionary. // Method to iterate all words in the dictionary.
// The returned token has to be used to get the next word. If token is 0, this method newly // The returned token has to be used to get the next word. If token is 0, this method newly

20
native/jni/src/suggest/core/session/prev_words_info.h
View File

@ -21,6 +21,7 @@
#include "suggest/core/dictionary/binary_dictionary_bigrams_iterator.h" #include "suggest/core/dictionary/binary_dictionary_bigrams_iterator.h"
#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h" #include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
#include "utils/char_utils.h" #include "utils/char_utils.h"
#include "utils/int_array_view.h"
namespace latinime { namespace latinime {
@ -91,19 +92,11 @@ class PrevWordsInfo {
} }
// n is 1-indexed. // n is 1-indexed.
const int *getNthPrevWordCodePoints(const int n) const { const CodePointArrayView getNthPrevWordCodePoints(const int n) const {
if (n <= 0 || n > MAX_PREV_WORD_COUNT_FOR_N_GRAM) { if (n <= 0 || n > MAX_PREV_WORD_COUNT_FOR_N_GRAM) {
return nullptr; return CodePointArrayView();
} }
return mPrevWordCodePoints[n - 1]; return CodePointArrayView(mPrevWordCodePoints[n - 1], mPrevWordCodePointCount[n - 1]);
}
// n is 1-indexed.
int getNthPrevWordCodePointCount(const int n) const {
if (n <= 0 || n > MAX_PREV_WORD_COUNT_FOR_N_GRAM) {
return 0;
}
return mPrevWordCodePointCount[n - 1];
} }
// n is 1-indexed. // n is 1-indexed.
@ -134,8 +127,9 @@ class PrevWordsInfo {
return NOT_A_DICT_POS; return NOT_A_DICT_POS;
} }
} }
const CodePointArrayView codePointArrayView(codePoints, codePointCount);
const int wordPtNodePos = dictStructurePolicy->getTerminalPtNodePositionOfWord( const int wordPtNodePos = dictStructurePolicy->getTerminalPtNodePositionOfWord(
codePoints, codePointCount, false /* forceLowerCaseSearch */); codePointArrayView, false /* forceLowerCaseSearch */);
if (wordPtNodePos != NOT_A_DICT_POS || !tryLowerCaseSearch) { if (wordPtNodePos != NOT_A_DICT_POS || !tryLowerCaseSearch) {
// Return the position when when the word was found or doesn't try lower case // Return the position when when the word was found or doesn't try lower case
// search. // search.
@ -144,7 +138,7 @@ class PrevWordsInfo {
// Check bigrams for lower-cased previous word if original was not found. Useful for // Check bigrams for lower-cased previous word if original was not found. Useful for
// auto-capitalized words like "The [current_word]". // auto-capitalized words like "The [current_word]".
return dictStructurePolicy->getTerminalPtNodePositionOfWord( return dictStructurePolicy->getTerminalPtNodePositionOfWord(
codePoints, codePointCount, true /* forceLowerCaseSearch */); codePointArrayView, true /* forceLowerCaseSearch */);
} }
void clear() { void clear() {

50
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.cpp
View File

@ -104,12 +104,12 @@ int Ver4PatriciaTriePolicy::getCodePointsAndProbabilityAndReturnCodePointCount(
return codePointCount; return codePointCount;
} }
int Ver4PatriciaTriePolicy::getTerminalPtNodePositionOfWord(const int *const inWord, int Ver4PatriciaTriePolicy::getTerminalPtNodePositionOfWord(const CodePointArrayView wordCodePoints,
const int length, const bool forceLowerCaseSearch) const { const bool forceLowerCaseSearch) const {
DynamicPtReadingHelper readingHelper(&mNodeReader, &mPtNodeArrayReader); DynamicPtReadingHelper readingHelper(&mNodeReader, &mPtNodeArrayReader);
readingHelper.initWithPtNodeArrayPos(getRootPosition()); readingHelper.initWithPtNodeArrayPos(getRootPosition());
const int ptNodePos = const int ptNodePos = readingHelper.getTerminalPtNodePositionOfWord(wordCodePoints.data(),
readingHelper.getTerminalPtNodePositionOfWord(inWord, length, forceLowerCaseSearch); wordCodePoints.size(), forceLowerCaseSearch);
if (readingHelper.isError()) { if (readingHelper.isError()) {
mIsCorrupted = true; mIsCorrupted = true;
AKLOGE("Dictionary reading error in createAndGetAllChildDicNodes()."); AKLOGE("Dictionary reading error in createAndGetAllChildDicNodes().");
@ -194,7 +194,7 @@ int Ver4PatriciaTriePolicy::getBigramsPositionOfPtNode(const int ptNodePos) cons
ptNodeParams.getTerminalId()); ptNodeParams.getTerminalId());
} }
bool Ver4PatriciaTriePolicy::addUnigramEntry(const int *const word, const int length, bool Ver4PatriciaTriePolicy::addUnigramEntry(const CodePointArrayView wordCodePoints,
const UnigramProperty *const unigramProperty) { const UnigramProperty *const unigramProperty) {
if (!mBuffers->isUpdatable()) { if (!mBuffers->isUpdatable()) {
AKLOGI("Warning: addUnigramEntry() is called for non-updatable dictionary."); AKLOGI("Warning: addUnigramEntry() is called for non-updatable dictionary.");
@ -205,8 +205,9 @@ bool Ver4PatriciaTriePolicy::addUnigramEntry(const int *const word, const int le
mDictBuffer->getTailPosition()); mDictBuffer->getTailPosition());
return false; return false;
} }
if (length > MAX_WORD_LENGTH) { if (wordCodePoints.size() > MAX_WORD_LENGTH) {
AKLOGE("The word is too long to insert to the dictionary, length: %d", length); AKLOGE("The word is too long to insert to the dictionary, length: %zd",
wordCodePoints.size());
return false; return false;
} }
for (const auto &shortcut : unigramProperty->getShortcuts()) { for (const auto &shortcut : unigramProperty->getShortcuts()) {
@ -220,8 +221,8 @@ bool Ver4PatriciaTriePolicy::addUnigramEntry(const int *const word, const int le
readingHelper.initWithPtNodeArrayPos(getRootPosition()); readingHelper.initWithPtNodeArrayPos(getRootPosition());
bool addedNewUnigram = false; bool addedNewUnigram = false;
int codePointsToAdd[MAX_WORD_LENGTH]; int codePointsToAdd[MAX_WORD_LENGTH];
int codePointCountToAdd = length; int codePointCountToAdd = wordCodePoints.size();
memmove(codePointsToAdd, word, sizeof(int) * length); memmove(codePointsToAdd, wordCodePoints.data(), sizeof(int) * codePointCountToAdd);
if (unigramProperty->representsBeginningOfSentence()) { if (unigramProperty->representsBeginningOfSentence()) {
codePointCountToAdd = CharUtils::attachBeginningOfSentenceMarker(codePointsToAdd, codePointCountToAdd = CharUtils::attachBeginningOfSentenceMarker(codePointsToAdd,
codePointCountToAdd, MAX_WORD_LENGTH); codePointCountToAdd, MAX_WORD_LENGTH);
@ -229,14 +230,15 @@ bool Ver4PatriciaTriePolicy::addUnigramEntry(const int *const word, const int le
if (codePointCountToAdd <= 0) { if (codePointCountToAdd <= 0) {
return false; return false;
} }
if (mUpdatingHelper.addUnigramWord(&readingHelper, codePointsToAdd, codePointCountToAdd, const CodePointArrayView codePointArrayView(codePointsToAdd, codePointCountToAdd);
unigramProperty, &addedNewUnigram)) { if (mUpdatingHelper.addUnigramWord(&readingHelper, codePointArrayView.data(),
codePointArrayView.size(), unigramProperty, &addedNewUnigram)) {
if (addedNewUnigram && !unigramProperty->representsBeginningOfSentence()) { if (addedNewUnigram && !unigramProperty->representsBeginningOfSentence()) {
mUnigramCount++; mUnigramCount++;
} }
if (unigramProperty->getShortcuts().size() > 0) { if (unigramProperty->getShortcuts().size() > 0) {
// Add shortcut target. // Add shortcut target.
const int wordPos = getTerminalPtNodePositionOfWord(word, length, const int wordPos = getTerminalPtNodePositionOfWord(codePointArrayView,
false /* forceLowerCaseSearch */); false /* forceLowerCaseSearch */);
if (wordPos == NOT_A_DICT_POS) { if (wordPos == NOT_A_DICT_POS) {
AKLOGE("Cannot find terminal PtNode position to add shortcut target."); AKLOGE("Cannot find terminal PtNode position to add shortcut target.");
@ -259,12 +261,12 @@ bool Ver4PatriciaTriePolicy::addUnigramEntry(const int *const word, const int le
} }
} }
bool Ver4PatriciaTriePolicy::removeUnigramEntry(const int *const word, const int length) { bool Ver4PatriciaTriePolicy::removeUnigramEntry(const CodePointArrayView wordCodePoints) {
if (!mBuffers->isUpdatable()) { if (!mBuffers->isUpdatable()) {
AKLOGI("Warning: removeUnigramEntry() is called for non-updatable dictionary."); AKLOGI("Warning: removeUnigramEntry() is called for non-updatable dictionary.");
return false; return false;
} }
const int ptNodePos = getTerminalPtNodePositionOfWord(word, length, const int ptNodePos = getTerminalPtNodePositionOfWord(wordCodePoints,
false /* forceLowerCaseSearch */); false /* forceLowerCaseSearch */);
if (ptNodePos == NOT_A_DICT_POS) { if (ptNodePos == NOT_A_DICT_POS) {
return false; return false;
@ -305,7 +307,6 @@ bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsI
false /* isBlacklisted */, MAX_PROBABILITY /* probability */, false /* isBlacklisted */, MAX_PROBABILITY /* probability */,
NOT_A_TIMESTAMP /* timestamp */, 0 /* level */, 0 /* count */, &shortcuts); NOT_A_TIMESTAMP /* timestamp */, 0 /* level */, 0 /* count */, &shortcuts);
if (!addUnigramEntry(prevWordsInfo->getNthPrevWordCodePoints(1 /* n */), if (!addUnigramEntry(prevWordsInfo->getNthPrevWordCodePoints(1 /* n */),
prevWordsInfo->getNthPrevWordCodePointCount(1 /* n */),
&beginningOfSentenceUnigramProperty)) { &beginningOfSentenceUnigramProperty)) {
AKLOGE("Cannot add unigram entry for the beginning-of-sentence."); AKLOGE("Cannot add unigram entry for the beginning-of-sentence.");
return false; return false;
@ -318,8 +319,8 @@ bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsI
} }
} }
const int word1Pos = getTerminalPtNodePositionOfWord( const int word1Pos = getTerminalPtNodePositionOfWord(
bigramProperty->getTargetCodePoints()->data(), CodePointArrayView(*bigramProperty->getTargetCodePoints()),
bigramProperty->getTargetCodePoints()->size(), false /* forceLowerCaseSearch */); false /* forceLowerCaseSearch */);
if (word1Pos == NOT_A_DICT_POS) { if (word1Pos == NOT_A_DICT_POS) {
return false; return false;
} }
@ -336,7 +337,7 @@ bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsI
} }
bool Ver4PatriciaTriePolicy::removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, bool Ver4PatriciaTriePolicy::removeNgramEntry(const PrevWordsInfo *const prevWordsInfo,
const int *const word, const int length) { const CodePointArrayView wordCodePoints) {
if (!mBuffers->isUpdatable()) { if (!mBuffers->isUpdatable()) {
AKLOGI("Warning: removeNgramEntry() is called for non-updatable dictionary."); AKLOGI("Warning: removeNgramEntry() is called for non-updatable dictionary.");
return false; return false;
@ -350,8 +351,9 @@ bool Ver4PatriciaTriePolicy::removeNgramEntry(const PrevWordsInfo *const prevWor
AKLOGE("prev words info is not valid for removing n-gram entry form the dictionary."); AKLOGE("prev words info is not valid for removing n-gram entry form the dictionary.");
return false; return false;
} }
if (length > MAX_WORD_LENGTH) { if (wordCodePoints.size() > MAX_WORD_LENGTH) {
AKLOGE("word is too long to remove n-gram entry form the dictionary. length: %d", length); AKLOGE("word is too long to remove n-gram entry form the dictionary. length: %zd",
wordCodePoints.size());
} }
int prevWordsPtNodePos[MAX_PREV_WORD_COUNT_FOR_N_GRAM]; int prevWordsPtNodePos[MAX_PREV_WORD_COUNT_FOR_N_GRAM];
prevWordsInfo->getPrevWordsTerminalPtNodePos(this, prevWordsPtNodePos, prevWordsInfo->getPrevWordsTerminalPtNodePos(this, prevWordsPtNodePos,
@ -360,7 +362,7 @@ bool Ver4PatriciaTriePolicy::removeNgramEntry(const PrevWordsInfo *const prevWor
if (prevWordsPtNodePos[0] == NOT_A_DICT_POS) { if (prevWordsPtNodePos[0] == NOT_A_DICT_POS) {
return false; return false;
} }
const int wordPos = getTerminalPtNodePositionOfWord(word, length, const int wordPos = getTerminalPtNodePositionOfWord(wordCodePoints,
false /* forceLowerCaseSearch */); false /* forceLowerCaseSearch */);
if (wordPos == NOT_A_DICT_POS) { if (wordPos == NOT_A_DICT_POS) {
return false; return false;
@ -445,9 +447,9 @@ void Ver4PatriciaTriePolicy::getProperty(const char *const query, const int quer
} }
} }
const WordProperty Ver4PatriciaTriePolicy::getWordProperty(const int *const codePoints, const WordProperty Ver4PatriciaTriePolicy::getWordProperty(
const int codePointCount) const { const CodePointArrayView wordCodePoints) const {
const int ptNodePos = getTerminalPtNodePositionOfWord(codePoints, codePointCount, const int ptNodePos = getTerminalPtNodePositionOfWord(wordCodePoints,
false /* forceLowerCaseSearch */); false /* forceLowerCaseSearch */);
if (ptNodePos == NOT_A_DICT_POS) { if (ptNodePos == NOT_A_DICT_POS) {
AKLOGE("getWordProperty is called for invalid word."); AKLOGE("getWordProperty is called for invalid word.");

18
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h
View File

@ -39,6 +39,7 @@
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_writing_helper.h" #include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_writing_helper.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_pt_node_array_reader.h" #include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_pt_node_array_reader.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "utils/int_array_view.h"
namespace latinime { namespace latinime {
namespace backward { namespace backward {
@ -75,7 +76,7 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
mBigramCount(mHeaderPolicy->getBigramCount()), mBigramCount(mHeaderPolicy->getBigramCount()),
mTerminalPtNodePositionsForIteratingWords(), mIsCorrupted(false) {}; mTerminalPtNodePositionsForIteratingWords(), mIsCorrupted(false) {};
AK_FORCE_INLINE int getRootPosition() const { virtual int getRootPosition() const {
return 0; return 0;
} }
@ -86,8 +87,8 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
const int terminalPtNodePos, const int maxCodePointCount, int *const outCodePoints, const int terminalPtNodePos, const int maxCodePointCount, int *const outCodePoints,
int *const outUnigramProbability) const; int *const outUnigramProbability) const;
int getTerminalPtNodePositionOfWord(const int *const inWord, int getTerminalPtNodePositionOfWord(const CodePointArrayView wordCodePoints,
const int length, const bool forceLowerCaseSearch) const; const bool forceLowerCaseSearch) const;
int getProbability(const int unigramProbability, const int bigramProbability) const; int getProbability(const int unigramProbability, const int bigramProbability) const;
@ -106,16 +107,16 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
return &mShortcutPolicy; return &mShortcutPolicy;
} }
bool addUnigramEntry(const int *const word, const int length, bool addUnigramEntry(const CodePointArrayView wordCodePoints,
const UnigramProperty *const unigramProperty); const UnigramProperty *const unigramProperty);
bool removeUnigramEntry(const int *const word, const int length); bool removeUnigramEntry(const CodePointArrayView wordCodePoints);
bool addNgramEntry(const PrevWordsInfo *const prevWordsInfo, bool addNgramEntry(const PrevWordsInfo *const prevWordsInfo,
const BigramProperty *const bigramProperty); const BigramProperty *const bigramProperty);
bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, const int *const word1, bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo,
const int length1); const CodePointArrayView wordCodePoints);
bool flush(const char *const filePath); bool flush(const char *const filePath);
@ -126,8 +127,7 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
void getProperty(const char *const query, const int queryLength, char *const outResult, void getProperty(const char *const query, const int queryLength, char *const outResult,
const int maxResultLength); const int maxResultLength);
const WordProperty getWordProperty(const int *const codePoints, const WordProperty getWordProperty(const CodePointArrayView wordCodePoints) const;
const int codePointCount) const;
int getNextWordAndNextToken(const int token, int *const outCodePoints, int getNextWordAndNextToken(const int token, int *const outCodePoints,
int *const outCodePointCount); int *const outCodePointCount);

14
native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.cpp
View File

@ -268,12 +268,12 @@ int PatriciaTriePolicy::getCodePointsAndProbabilityAndReturnCodePointCount(
// This function gets the position of the terminal PtNode of the exact matching word in the // This function gets the position of the terminal PtNode of the exact matching word in the
// dictionary. If no match is found, it returns NOT_A_DICT_POS. // dictionary. If no match is found, it returns NOT_A_DICT_POS.
int PatriciaTriePolicy::getTerminalPtNodePositionOfWord(const int *const inWord, int PatriciaTriePolicy::getTerminalPtNodePositionOfWord(const CodePointArrayView wordCodePoints,
const int length, const bool forceLowerCaseSearch) const { const bool forceLowerCaseSearch) const {
DynamicPtReadingHelper readingHelper(&mPtNodeReader, &mPtNodeArrayReader); DynamicPtReadingHelper readingHelper(&mPtNodeReader, &mPtNodeArrayReader);
readingHelper.initWithPtNodeArrayPos(getRootPosition()); readingHelper.initWithPtNodeArrayPos(getRootPosition());
const int ptNodePos = const int ptNodePos = readingHelper.getTerminalPtNodePositionOfWord(wordCodePoints.data(),
readingHelper.getTerminalPtNodePositionOfWord(inWord, length, forceLowerCaseSearch); wordCodePoints.size(), forceLowerCaseSearch);
if (readingHelper.isError()) { if (readingHelper.isError()) {
mIsCorrupted = true; mIsCorrupted = true;
AKLOGE("Dictionary reading error in createAndGetAllChildDicNodes()."); AKLOGE("Dictionary reading error in createAndGetAllChildDicNodes().");
@ -377,9 +377,9 @@ int PatriciaTriePolicy::createAndGetLeavingChildNode(const DicNode *const dicNod
return siblingPos; return siblingPos;
} }
const WordProperty PatriciaTriePolicy::getWordProperty(const int *const codePoints, const WordProperty PatriciaTriePolicy::getWordProperty(
const int codePointCount) const { const CodePointArrayView wordCodePoints) const {
const int ptNodePos = getTerminalPtNodePositionOfWord(codePoints, codePointCount, const int ptNodePos = getTerminalPtNodePositionOfWord(wordCodePoints,
false /* forceLowerCaseSearch */); false /* forceLowerCaseSearch */);
if (ptNodePos == NOT_A_DICT_POS) { if (ptNodePos == NOT_A_DICT_POS) {
AKLOGE("getWordProperty was called for invalid word."); AKLOGE("getWordProperty was called for invalid word.");

16
native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h
View File

@ -30,6 +30,7 @@
#include "suggest/policyimpl/dictionary/utils/format_utils.h" #include "suggest/policyimpl/dictionary/utils/format_utils.h"
#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h" #include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
#include "utils/byte_array_view.h" #include "utils/byte_array_view.h"
#include "utils/int_array_view.h"
namespace latinime { namespace latinime {
@ -63,8 +64,8 @@ class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
const int terminalNodePos, const int maxCodePointCount, int *const outCodePoints, const int terminalNodePos, const int maxCodePointCount, int *const outCodePoints,
int *const outUnigramProbability) const; int *const outUnigramProbability) const;
int getTerminalPtNodePositionOfWord(const int *const inWord, int getTerminalPtNodePositionOfWord(const CodePointArrayView wordCodePoints,
const int length, const bool forceLowerCaseSearch) const; const bool forceLowerCaseSearch) const;
int getProbability(const int unigramProbability, const int bigramProbability) const; int getProbability(const int unigramProbability, const int bigramProbability) const;
@ -83,14 +84,14 @@ class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
return &mShortcutListPolicy; return &mShortcutListPolicy;
} }
bool addUnigramEntry(const int *const word, const int length, bool addUnigramEntry(const CodePointArrayView wordCodePoints,
const UnigramProperty *const unigramProperty) { const UnigramProperty *const unigramProperty) {
// This method should not be called for non-updatable dictionary. // This method should not be called for non-updatable dictionary.
AKLOGI("Warning: addUnigramEntry() is called for non-updatable dictionary."); AKLOGI("Warning: addUnigramEntry() is called for non-updatable dictionary.");
return false; return false;
} }
bool removeUnigramEntry(const int *const word, const int length) { bool removeUnigramEntry(const CodePointArrayView wordCodePoints) {
// This method should not be called for non-updatable dictionary. // This method should not be called for non-updatable dictionary.
AKLOGI("Warning: removeUnigramEntry() is called for non-updatable dictionary."); AKLOGI("Warning: removeUnigramEntry() is called for non-updatable dictionary.");
return false; return false;
@ -103,8 +104,8 @@ class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
return false; return false;
} }
bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, const int *const word, bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo,
const int length) { const CodePointArrayView wordCodePoints) {
// This method should not be called for non-updatable dictionary. // This method should not be called for non-updatable dictionary.
AKLOGI("Warning: removeNgramEntry() is called for non-updatable dictionary."); AKLOGI("Warning: removeNgramEntry() is called for non-updatable dictionary.");
return false; return false;
@ -136,8 +137,7 @@ class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
} }
} }
const WordProperty getWordProperty(const int *const codePoints, const WordProperty getWordProperty(const CodePointArrayView wordCodePoints) const;
const int codePointCount) const;
int getNextWordAndNextToken(const int token, int *const outCodePoints, int getNextWordAndNextToken(const int token, int *const outCodePoints,
int *const outCodePointCount); int *const outCodePointCount);

50
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
View File

@ -94,12 +94,12 @@ int Ver4PatriciaTriePolicy::getCodePointsAndProbabilityAndReturnCodePointCount(
return codePointCount; return codePointCount;
} }
int Ver4PatriciaTriePolicy::getTerminalPtNodePositionOfWord(const int *const inWord, int Ver4PatriciaTriePolicy::getTerminalPtNodePositionOfWord(const CodePointArrayView wordCodePoints,
const int length, const bool forceLowerCaseSearch) const { const bool forceLowerCaseSearch) const {
DynamicPtReadingHelper readingHelper(&mNodeReader, &mPtNodeArrayReader); DynamicPtReadingHelper readingHelper(&mNodeReader, &mPtNodeArrayReader);
readingHelper.initWithPtNodeArrayPos(getRootPosition()); readingHelper.initWithPtNodeArrayPos(getRootPosition());
const int ptNodePos = const int ptNodePos = readingHelper.getTerminalPtNodePositionOfWord(wordCodePoints.data(),
readingHelper.getTerminalPtNodePositionOfWord(inWord, length, forceLowerCaseSearch); wordCodePoints.size(), forceLowerCaseSearch);
if (readingHelper.isError()) { if (readingHelper.isError()) {
mIsCorrupted = true; mIsCorrupted = true;
AKLOGE("Dictionary reading error in createAndGetAllChildDicNodes()."); AKLOGE("Dictionary reading error in createAndGetAllChildDicNodes().");
@ -189,7 +189,7 @@ int Ver4PatriciaTriePolicy::getShortcutPositionOfPtNode(const int ptNodePos) con
ptNodeParams.getTerminalId()); ptNodeParams.getTerminalId());
} }
bool Ver4PatriciaTriePolicy::addUnigramEntry(const int *const word, const int length, bool Ver4PatriciaTriePolicy::addUnigramEntry(const CodePointArrayView wordCodePoints,
const UnigramProperty *const unigramProperty) { const UnigramProperty *const unigramProperty) {
if (!mBuffers->isUpdatable()) { if (!mBuffers->isUpdatable()) {
AKLOGI("Warning: addUnigramEntry() is called for non-updatable dictionary."); AKLOGI("Warning: addUnigramEntry() is called for non-updatable dictionary.");
@ -200,8 +200,9 @@ bool Ver4PatriciaTriePolicy::addUnigramEntry(const int *const word, const int le
mDictBuffer->getTailPosition()); mDictBuffer->getTailPosition());
return false; return false;
} }
if (length > MAX_WORD_LENGTH) { if (wordCodePoints.size() > MAX_WORD_LENGTH) {
AKLOGE("The word is too long to insert to the dictionary, length: %d", length); AKLOGE("The word is too long to insert to the dictionary, length: %zd",
wordCodePoints.size());
return false; return false;
} }
for (const auto &shortcut : unigramProperty->getShortcuts()) { for (const auto &shortcut : unigramProperty->getShortcuts()) {
@ -215,8 +216,8 @@ bool Ver4PatriciaTriePolicy::addUnigramEntry(const int *const word, const int le
readingHelper.initWithPtNodeArrayPos(getRootPosition()); readingHelper.initWithPtNodeArrayPos(getRootPosition());
bool addedNewUnigram = false; bool addedNewUnigram = false;
int codePointsToAdd[MAX_WORD_LENGTH]; int codePointsToAdd[MAX_WORD_LENGTH];
int codePointCountToAdd = length; int codePointCountToAdd = wordCodePoints.size();
memmove(codePointsToAdd, word, sizeof(int) * length); memmove(codePointsToAdd, wordCodePoints.data(), sizeof(int) * codePointCountToAdd);
if (unigramProperty->representsBeginningOfSentence()) { if (unigramProperty->representsBeginningOfSentence()) {
codePointCountToAdd = CharUtils::attachBeginningOfSentenceMarker(codePointsToAdd, codePointCountToAdd = CharUtils::attachBeginningOfSentenceMarker(codePointsToAdd,
codePointCountToAdd, MAX_WORD_LENGTH); codePointCountToAdd, MAX_WORD_LENGTH);
@ -224,14 +225,15 @@ bool Ver4PatriciaTriePolicy::addUnigramEntry(const int *const word, const int le
if (codePointCountToAdd <= 0) { if (codePointCountToAdd <= 0) {
return false; return false;
} }
if (mUpdatingHelper.addUnigramWord(&readingHelper, codePointsToAdd, codePointCountToAdd, const CodePointArrayView codePointArrayView(codePointsToAdd, codePointCountToAdd);
unigramProperty, &addedNewUnigram)) { if (mUpdatingHelper.addUnigramWord(&readingHelper, codePointArrayView.data(),
codePointArrayView.size(), unigramProperty, &addedNewUnigram)) {
if (addedNewUnigram && !unigramProperty->representsBeginningOfSentence()) { if (addedNewUnigram && !unigramProperty->representsBeginningOfSentence()) {
mUnigramCount++; mUnigramCount++;
} }
if (unigramProperty->getShortcuts().size() > 0) { if (unigramProperty->getShortcuts().size() > 0) {
// Add shortcut target. // Add shortcut target.
const int wordPos = getTerminalPtNodePositionOfWord(word, length, const int wordPos = getTerminalPtNodePositionOfWord(codePointArrayView,
false /* forceLowerCaseSearch */); false /* forceLowerCaseSearch */);
if (wordPos == NOT_A_DICT_POS) { if (wordPos == NOT_A_DICT_POS) {
AKLOGE("Cannot find terminal PtNode position to add shortcut target."); AKLOGE("Cannot find terminal PtNode position to add shortcut target.");
@ -254,12 +256,12 @@ bool Ver4PatriciaTriePolicy::addUnigramEntry(const int *const word, const int le
} }
} }
bool Ver4PatriciaTriePolicy::removeUnigramEntry(const int *const word, const int length) { bool Ver4PatriciaTriePolicy::removeUnigramEntry(const CodePointArrayView wordCodePoints) {
if (!mBuffers->isUpdatable()) { if (!mBuffers->isUpdatable()) {
AKLOGI("Warning: removeUnigramEntry() is called for non-updatable dictionary."); AKLOGI("Warning: removeUnigramEntry() is called for non-updatable dictionary.");
return false; return false;
} }
const int ptNodePos = getTerminalPtNodePositionOfWord(word, length, const int ptNodePos = getTerminalPtNodePositionOfWord(wordCodePoints,
false /* forceLowerCaseSearch */); false /* forceLowerCaseSearch */);
if (ptNodePos == NOT_A_DICT_POS) { if (ptNodePos == NOT_A_DICT_POS) {
return false; return false;
@ -313,7 +315,6 @@ bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsI
false /* isBlacklisted */, MAX_PROBABILITY /* probability */, false /* isBlacklisted */, MAX_PROBABILITY /* probability */,
NOT_A_TIMESTAMP /* timestamp */, 0 /* level */, 0 /* count */, &shortcuts); NOT_A_TIMESTAMP /* timestamp */, 0 /* level */, 0 /* count */, &shortcuts);
if (!addUnigramEntry(prevWordsInfo->getNthPrevWordCodePoints(1 /* n */), if (!addUnigramEntry(prevWordsInfo->getNthPrevWordCodePoints(1 /* n */),
prevWordsInfo->getNthPrevWordCodePointCount(1 /* n */),
&beginningOfSentenceUnigramProperty)) { &beginningOfSentenceUnigramProperty)) {
AKLOGE("Cannot add unigram entry for the beginning-of-sentence."); AKLOGE("Cannot add unigram entry for the beginning-of-sentence.");
return false; return false;
@ -326,8 +327,8 @@ bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsI
} }
} }
const int word1Pos = getTerminalPtNodePositionOfWord( const int word1Pos = getTerminalPtNodePositionOfWord(
bigramProperty->getTargetCodePoints()->data(), CodePointArrayView(*bigramProperty->getTargetCodePoints()),
bigramProperty->getTargetCodePoints()->size(), false /* forceLowerCaseSearch */); false /* forceLowerCaseSearch */);
if (word1Pos == NOT_A_DICT_POS) { if (word1Pos == NOT_A_DICT_POS) {
return false; return false;
} }
@ -344,7 +345,7 @@ bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsI
} }
bool Ver4PatriciaTriePolicy::removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, bool Ver4PatriciaTriePolicy::removeNgramEntry(const PrevWordsInfo *const prevWordsInfo,
const int *const word, const int length) { const CodePointArrayView wordCodePoints) {
if (!mBuffers->isUpdatable()) { if (!mBuffers->isUpdatable()) {
AKLOGI("Warning: removeNgramEntry() is called for non-updatable dictionary."); AKLOGI("Warning: removeNgramEntry() is called for non-updatable dictionary.");
return false; return false;
@ -358,8 +359,9 @@ bool Ver4PatriciaTriePolicy::removeNgramEntry(const PrevWordsInfo *const prevWor
AKLOGE("prev words info is not valid for removing n-gram entry form the dictionary."); AKLOGE("prev words info is not valid for removing n-gram entry form the dictionary.");
return false; return false;
} }
if (length > MAX_WORD_LENGTH) { if (wordCodePoints.size() > MAX_WORD_LENGTH) {
AKLOGE("word is too long to remove n-gram entry form the dictionary. length: %d", length); AKLOGE("word is too long to remove n-gram entry form the dictionary. length: %zd",
wordCodePoints.size());
} }
int prevWordsPtNodePos[MAX_PREV_WORD_COUNT_FOR_N_GRAM]; int prevWordsPtNodePos[MAX_PREV_WORD_COUNT_FOR_N_GRAM];
prevWordsInfo->getPrevWordsTerminalPtNodePos(this, prevWordsPtNodePos, prevWordsInfo->getPrevWordsTerminalPtNodePos(this, prevWordsPtNodePos,
@ -369,7 +371,7 @@ bool Ver4PatriciaTriePolicy::removeNgramEntry(const PrevWordsInfo *const prevWor
if (prevWordsPtNodePos[0] == NOT_A_DICT_POS) { if (prevWordsPtNodePos[0] == NOT_A_DICT_POS) {
return false; return false;
} }
const int wordPos = getTerminalPtNodePositionOfWord(word, length, const int wordPos = getTerminalPtNodePositionOfWord(wordCodePoints,
false /* forceLowerCaseSearch */); false /* forceLowerCaseSearch */);
if (wordPos == NOT_A_DICT_POS) { if (wordPos == NOT_A_DICT_POS) {
return false; return false;
@ -453,9 +455,9 @@ void Ver4PatriciaTriePolicy::getProperty(const char *const query, const int quer
} }
} }
const WordProperty Ver4PatriciaTriePolicy::getWordProperty(const int *const codePoints, const WordProperty Ver4PatriciaTriePolicy::getWordProperty(
const int codePointCount) const { const CodePointArrayView wordCodePoints) const {
const int ptNodePos = getTerminalPtNodePositionOfWord(codePoints, codePointCount, const int ptNodePos = getTerminalPtNodePositionOfWord(wordCodePoints,
false /* forceLowerCaseSearch */); false /* forceLowerCaseSearch */);
if (ptNodePos == NOT_A_DICT_POS) { if (ptNodePos == NOT_A_DICT_POS) {
AKLOGE("getWordProperty is called for invalid word."); AKLOGE("getWordProperty is called for invalid word.");

16
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
View File

@ -30,6 +30,7 @@
#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_pt_node_array_reader.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_pt_node_array_reader.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "utils/int_array_view.h"
namespace latinime { namespace latinime {
@ -65,8 +66,8 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
const int terminalPtNodePos, const int maxCodePointCount, int *const outCodePoints, const int terminalPtNodePos, const int maxCodePointCount, int *const outCodePoints,
int *const outUnigramProbability) const; int *const outUnigramProbability) const;
int getTerminalPtNodePositionOfWord(const int *const inWord, int getTerminalPtNodePositionOfWord(const CodePointArrayView wordCodePoints,
const int length, const bool forceLowerCaseSearch) const; const bool forceLowerCaseSearch) const;
int getProbability(const int unigramProbability, const int bigramProbability) const; int getProbability(const int unigramProbability, const int bigramProbability) const;
@ -85,16 +86,16 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
return &mShortcutPolicy; return &mShortcutPolicy;
} }
bool addUnigramEntry(const int *const word, const int length, bool addUnigramEntry(const CodePointArrayView wordCodePoints,
const UnigramProperty *const unigramProperty); const UnigramProperty *const unigramProperty);
bool removeUnigramEntry(const int *const word, const int length); bool removeUnigramEntry(const CodePointArrayView wordCodePoints);
bool addNgramEntry(const PrevWordsInfo *const prevWordsInfo, bool addNgramEntry(const PrevWordsInfo *const prevWordsInfo,
const BigramProperty *const bigramProperty); const BigramProperty *const bigramProperty);
bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, const int *const word1, bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo,
const int length1); const CodePointArrayView wordCodePoints);
bool flush(const char *const filePath); bool flush(const char *const filePath);
@ -105,8 +106,7 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
void getProperty(const char *const query, const int queryLength, char *const outResult, void getProperty(const char *const query, const int queryLength, char *const outResult,
const int maxResultLength); const int maxResultLength);
const WordProperty getWordProperty(const int *const codePoints, const WordProperty getWordProperty(const CodePointArrayView wordCodePoints) const;
const int codePointCount) const;
int getNextWordAndNextToken(const int token, int *const outCodePoints, int getNextWordAndNextToken(const int token, int *const outCodePoints,
int *const outCodePointCount); int *const outCodePointCount);

1
native/jni/src/utils/int_array_view.h
View File

@ -105,6 +105,7 @@ class IntArrayView {
using WordIdArrayView = IntArrayView; using WordIdArrayView = IntArrayView;
using PtNodePosArrayView = IntArrayView; using PtNodePosArrayView = IntArrayView;
using CodePointArrayView = IntArrayView;
} // namespace latinime } // namespace latinime
#endif // LATINIME_MEMORY_VIEW_H #endif // LATINIME_MEMORY_VIEW_H