diff --git a/native/jni/Android.mk b/native/jni/Android.mk index f5c00f84b..7511f4efa 100644 --- a/native/jni/Android.mk +++ b/native/jni/Android.mk @@ -99,6 +99,7 @@ LATIN_IME_CORE_SRC_FILES := \ ver4_patricia_trie_writing_helper.cpp) \ $(addprefix suggest/policyimpl/dictionary/structure/v4/content/, \ bigram_dict_content.cpp \ + probability_dict_content.cpp \ shortcut_dict_content.cpp \ sparse_table_dict_content.cpp \ terminal_position_lookup_table.cpp) \ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_helper.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_helper.h index 4a6b964ef..e343bf9e5 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_helper.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_helper.h @@ -17,6 +17,7 @@ #ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H #define LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H +#include #include #include "defines.h" diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_contect.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_contect.cpp new file mode 100644 index 000000000..eab344e99 --- /dev/null +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_contect.cpp @@ -0,0 +1,96 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h" + +#include "suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h" +#include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h" +#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" +#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.h" +#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" + +namespace latinime { + +int ProbabilityDictContent::getProbability(const int terminalId) const { + if (terminalId < 0 || terminalId >= mSize) { + return NOT_A_PROBABILITY; + } + return Ver4PatriciaTrieReadingUtils::getProbability(getBuffer(), terminalId); +} + +bool ProbabilityDictContent::setProbability(const int terminalId, const int probability) { + if (terminalId < 0) { + return false; + } + if (terminalId >= mSize) { + // Write new entry. + int writingPos = getBuffer()->getTailPosition(); + while (writingPos <= getEntryPos(terminalId)) { + const int dummyFlags = 0; + if (!getWritableBuffer()->writeUintAndAdvancePosition(dummyFlags, + Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE, &writingPos)) { + return false; + } + const int dummyProbability = 0; + if (!getWritableBuffer()->writeUintAndAdvancePosition(dummyProbability, + Ver4DictConstants::PROBABILITY_SIZE, &writingPos)) { + return false; + } + mSize++; + } + } + const int probabilityWritingPos = getEntryPos(terminalId) + + Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE; + return getWritableBuffer()->writeUint(probability, + Ver4DictConstants::PROBABILITY_SIZE, probabilityWritingPos); +} + +bool ProbabilityDictContent::flushToFile(const char *const dictDirPath) const { + if (getEntryPos(mSize) < getBuffer()->getTailPosition()) { + ProbabilityDictContent probabilityDictContentToWrite; + for (int i = 0; i < mSize; ++i) { + if (!probabilityDictContentToWrite.setProbability(i, getProbability(i))) { + return false; + } + } + return probabilityDictContentToWrite.flush(dictDirPath, + Ver4DictConstants::FREQ_FILE_EXTENSION); + } else { + return flush(dictDirPath, Ver4DictConstants::FREQ_FILE_EXTENSION); + } +} + +bool ProbabilityDictContent::runGC( + const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap, + const ProbabilityDictContent *const originalProbabilityDictContent) { + mSize = 0; + for (TerminalPositionLookupTable::TerminalIdMap::const_iterator it = terminalIdMap->begin(); + it != terminalIdMap->end(); ++it) { + if (!setProbability(it->second, + originalProbabilityDictContent->getProbability(it->first))) { + return false; + } + mSize++; + } + return true; +} + +int ProbabilityDictContent::getEntryPos(const int terminalId) const { + return terminalId * (Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE + + Ver4DictConstants::PROBABILITY_SIZE); +} + +} // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h index 0958dd124..0971ee0e6 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h @@ -21,7 +21,6 @@ #include "suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h" #include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" -#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" namespace latinime { @@ -35,76 +34,19 @@ class ProbabilityDictContent : public SingleDictContent { ProbabilityDictContent() : mSize(0) {} - int getProbability(const int terminalId) const { - if (terminalId < 0 || terminalId >= mSize) { - return NOT_A_PROBABILITY; - } - return Ver4PatriciaTrieReadingUtils::getProbability(getBuffer(), terminalId); - } + int getProbability(const int terminalId) const; - bool setProbability(const int terminalId, const int probability) { - if (terminalId < 0) { - return false; - } - if (terminalId >= mSize) { - // Write new entry. - int writingPos = getBuffer()->getTailPosition(); - while (writingPos <= getEntryPos(terminalId)) { - const int dummyFlags = 0; - if (!getWritableBuffer()->writeUintAndAdvancePosition(dummyFlags, - Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE, &writingPos)) { - return false; - } - const int dummyProbability = 0; - if (!getWritableBuffer()->writeUintAndAdvancePosition(dummyProbability, - Ver4DictConstants::PROBABILITY_SIZE, &writingPos)) { - return false; - } - mSize++; - } - } - const int probabilityWritingPos = getEntryPos(terminalId) - + Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE; - return getWritableBuffer()->writeUint(probability, - Ver4DictConstants::PROBABILITY_SIZE, probabilityWritingPos); - } + bool setProbability(const int terminalId, const int probability); - bool flushToFile(const char *const dictDirPath) const { - if (getEntryPos(mSize) < getBuffer()->getTailPosition()) { - ProbabilityDictContent probabilityDictContentToWrite; - for (int i = 0; i < mSize; ++i) { - if (!probabilityDictContentToWrite.setProbability(i, getProbability(i))) { - return false; - } - } - return probabilityDictContentToWrite.flush(dictDirPath, - Ver4DictConstants::FREQ_FILE_EXTENSION); - } else { - return flush(dictDirPath, Ver4DictConstants::FREQ_FILE_EXTENSION); - } - } + bool flushToFile(const char *const dictDirPath) const; bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap, - const ProbabilityDictContent *const originalProbabilityDictContent) { - mSize = 0; - for (TerminalPositionLookupTable::TerminalIdMap::const_iterator it = terminalIdMap->begin(); - it != terminalIdMap->end(); ++it) { - if (!setProbability(it->second, - originalProbabilityDictContent->getProbability(it->first))) { - return false; - } - mSize++; - } - return true; - } + const ProbabilityDictContent *const originalProbabilityDictContent); private: DISALLOW_COPY_AND_ASSIGN(ProbabilityDictContent); - int getEntryPos(const int terminalId) const { - return terminalId * (Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE - + Ver4DictConstants::PROBABILITY_SIZE); - } + int getEntryPos(const int terminalId) const; int mSize; }; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp index b18bdc943..e9fba79d7 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp @@ -19,6 +19,7 @@ #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h" +#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" namespace latinime {