nyancrimew/LatinIME
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge "Purge dict reading code for 401 and create for 402."

Browse source
This commit is contained in:
Keisuke Kuroyanagi 2014-06-10 10:31:04 +00:00 committed by Android (Google) Code Review
commit 48b0074349
42 changed files with 333 additions and 321 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
View file

@ -115,8 +115,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
private boolean needsToMigrateDictionary(final int formatVersion) {
// When we bump up the dictionary format version, the old version should be added to here
// for supporting migration. Note that native code has to support reading such formats.
return formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING
|| formatVersion == FormatSpec.VERSION401;
return formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING;
}
public boolean isValidDictionaryLocked() {

10
native/jni/NativeFileList.mk
View file

@ -100,8 +100,8 @@ LATIN_IME_CORE_SRC_FILES := \
log_utils.cpp \
time_keeper.cpp)
LATIN_IME_CORE_SRC_FILES_BACKWARD_V401 := \
$(addprefix suggest/policyimpl/dictionary/structure/backward/v401/, \
LATIN_IME_CORE_SRC_FILES_BACKWARD_V402 := \
$(addprefix suggest/policyimpl/dictionary/structure/backward/v402/, \
ver4_dict_buffers.cpp \
ver4_dict_constants.cpp \
ver4_patricia_trie_node_reader.cpp \
@ -110,16 +110,16 @@ LATIN_IME_CORE_SRC_FILES_BACKWARD_V401 := \
ver4_patricia_trie_reading_utils.cpp \
ver4_patricia_trie_writing_helper.cpp \
ver4_pt_node_array_reader.cpp) \
$(addprefix suggest/policyimpl/dictionary/structure/backward/v401/content/, \
$(addprefix suggest/policyimpl/dictionary/structure/backward/v402/content/, \
bigram_dict_content.cpp \
probability_dict_content.cpp \
shortcut_dict_content.cpp \
sparse_table_dict_content.cpp \
terminal_position_lookup_table.cpp) \
$(addprefix suggest/policyimpl/dictionary/structure/backward/v401/bigram/, \
$(addprefix suggest/policyimpl/dictionary/structure/backward/v402/bigram/, \
ver4_bigram_list_policy.cpp)
LATIN_IME_CORE_SRC_FILES += $(LATIN_IME_CORE_SRC_FILES_BACKWARD_V401)
LATIN_IME_CORE_SRC_FILES += $(LATIN_IME_CORE_SRC_FILES_BACKWARD_V402)
LATIN_IME_CORE_TEST_FILES := \
defines_test.cpp \

4
native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
View file

@ -139,8 +139,6 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
switch (mDictFormatVersion) {
case FormatUtils::VERSION_2:
return FormatUtils::VERSION_2;
case FormatUtils::VERSION_401:
return FormatUtils::VERSION_401;
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
return FormatUtils::VERSION_4_ONLY_FOR_TESTING;
case FormatUtils::VERSION_4:
@ -249,7 +247,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
}
bool supportsBeginningOfSentence() const {
return mDictFormatVersion > FormatUtils::VERSION_401;
return mDictFormatVersion >= FormatUtils::VERSION_4;
}
private:

1
native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp
View file

@ -98,7 +98,6 @@ typedef DictionaryHeaderStructurePolicy::AttributeMap AttributeMap;
case FormatUtils::VERSION_2:
// Version 2 dictionary writing is not supported.
return false;
case FormatUtils::VERSION_401:
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
case FormatUtils::VERSION_4:
case FormatUtils::VERSION_4_DEV:

0
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/Readme.txt → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/Readme.txt
View file

12
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/bigram/ver4_bigram_list_policy.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/bigram/ver4_bigram_list_policy.cpp
View file

@ -22,19 +22,19 @@
* suggest/policyimpl/dictionary/structure/v4/bigram/ver4_bigram_list_policy.cpp
*/
#include "suggest/policyimpl/dictionary/structure/backward/v401/bigram/ver4_bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/bigram/ver4_bigram_list_policy.h"
#include "suggest/core/dictionary/property/bigram_property.h"
#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/bigram/bigram_list_read_write_utils.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
void Ver4BigramListPolicy::getNextBigram(int *const outBigramPos, int *const outProbability,
bool *const outHasNext, int *const bigramEntryPos) const {
@ -285,6 +285,6 @@ bool Ver4BigramListPolicy::updateHasNextFlag(const bool hasNext, const int bigra
return mBigramDictContent->writeBigramEntry(&updatedBigramEntry, bigramEntryPos);
}
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime

20
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/bigram/ver4_bigram_list_policy.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/bigram/ver4_bigram_list_policy.h
View file

@ -22,28 +22,28 @@
* suggest/policyimpl/dictionary/structure/v4/bigram/ver4_bigram_list_policy.h
*/
#ifndef LATINIME_BACKWARD_V401_VER4_BIGRAM_LIST_POLICY_H
#define LATINIME_BACKWARD_V401_VER4_BIGRAM_LIST_POLICY_H
#ifndef LATINIME_BACKWARD_V402_VER4_BIGRAM_LIST_POLICY_H
#define LATINIME_BACKWARD_V402_VER4_BIGRAM_LIST_POLICY_H
#include "defines.h"
#include "suggest/core/policy/dictionary_bigrams_structure_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_entry.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_entry.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
class BigramDictContent;
} // namespace v401
} // namespace v402
} // namespace backward
class BigramProperty;
namespace backward {
namespace v401 {
} // namespace v401
namespace v402 {
} // namespace v402
} // namespace backward
class HeaderPolicy;
namespace backward {
namespace v401 {
namespace v402 {
class TerminalPositionLookupTable;
class Ver4BigramListPolicy : public DictionaryBigramsStructurePolicy {
@ -87,7 +87,7 @@ class Ver4BigramListPolicy : public DictionaryBigramsStructurePolicy {
const TerminalPositionLookupTable *const mTerminalPositionLookupTable;
const HeaderPolicy *const mHeaderPolicy;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_VER4_BIGRAM_LIST_POLICY_H */
#endif /* LATINIME_BACKWARD_V402_VER4_BIGRAM_LIST_POLICY_H */

36
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_dict_content.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_dict_content.cpp
View file

@ -21,20 +21,22 @@
* suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.cpp
*/
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_dict_content.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition(
int *const bigramEntryPos) const {
const BufferWithExtendableBuffer *const bigramListBuffer = getContentBuffer();
if (*bigramEntryPos < 0 || *bigramEntryPos >= bigramListBuffer->getTailPosition()) {
AKLOGE("Invalid bigram entry position. bigramEntryPos: %d, bufSize: %d",
*bigramEntryPos, bigramListBuffer->getTailPosition());
const int bigramEntryTailPos = (*bigramEntryPos) + getBigramEntrySize();
if (*bigramEntryPos < 0 || bigramEntryTailPos > bigramListBuffer->getTailPosition()) {
AKLOGE("Invalid bigram entry position. bigramEntryPos: %d, bigramEntryTailPos: %d, "
"bufSize: %d", *bigramEntryPos, bigramEntryTailPos,
bigramListBuffer->getTailPosition());
ASSERT(false);
return BigramEntry(false /* hasNext */, NOT_A_PROBABILITY,
Ver4DictConstants::NOT_A_TERMINAL_ID);
@ -47,8 +49,6 @@ const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition(
int level = 0;
int count = 0;
if (mHasHistoricalInfo) {
probability = bigramListBuffer->readUintAndAdvancePosition(
Ver4DictConstants::PROBABILITY_SIZE, bigramEntryPos);
timestamp = bigramListBuffer->readUintAndAdvancePosition(
Ver4DictConstants::TIME_STAMP_FIELD_SIZE, bigramEntryPos);
level = bigramListBuffer->readUintAndAdvancePosition(
@ -56,7 +56,8 @@ const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition(
count = bigramListBuffer->readUintAndAdvancePosition(
Ver4DictConstants::WORD_COUNT_FIELD_SIZE, bigramEntryPos);
} else {
probability = bigramFlags & Ver4DictConstants::BIGRAM_PROBABILITY_MASK;
probability = bigramListBuffer->readUintAndAdvancePosition(
Ver4DictConstants::PROBABILITY_SIZE, bigramEntryPos);
}
const int encodedTargetTerminalId = bigramListBuffer->readUintAndAdvancePosition(
Ver4DictConstants::BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE, bigramEntryPos);
@ -74,21 +75,13 @@ const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition(
bool BigramDictContent::writeBigramEntryAndAdvancePosition(
const BigramEntry *const bigramEntryToWrite, int *const entryWritingPos) {
BufferWithExtendableBuffer *const bigramListBuffer = getWritableContentBuffer();
const int bigramFlags = createAndGetBigramFlags(
mHasHistoricalInfo ? 0 : bigramEntryToWrite->getProbability(),
bigramEntryToWrite->hasNext());
const int bigramFlags = createAndGetBigramFlags(bigramEntryToWrite->hasNext());
if (!bigramListBuffer->writeUintAndAdvancePosition(bigramFlags,
Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE, entryWritingPos)) {
AKLOGE("Cannot write bigram flags. pos: %d, flags: %x", *entryWritingPos, bigramFlags);
return false;
}
if (mHasHistoricalInfo) {
if (!bigramListBuffer->writeUintAndAdvancePosition(bigramEntryToWrite->getProbability(),
Ver4DictConstants::PROBABILITY_SIZE, entryWritingPos)) {
AKLOGE("Cannot write bigram probability. pos: %d, probability: %d", *entryWritingPos,
bigramEntryToWrite->getProbability());
return false;
}
const HistoricalInfo *const historicalInfo = bigramEntryToWrite->getHistoricalInfo();
if (!bigramListBuffer->writeUintAndAdvancePosition(historicalInfo->getTimeStamp(),
Ver4DictConstants::TIME_STAMP_FIELD_SIZE, entryWritingPos)) {
@ -108,6 +101,13 @@ bool BigramDictContent::writeBigramEntryAndAdvancePosition(
historicalInfo->getCount());
return false;
}
} else {
if (!bigramListBuffer->writeUintAndAdvancePosition(bigramEntryToWrite->getProbability(),
Ver4DictConstants::PROBABILITY_SIZE, entryWritingPos)) {
AKLOGE("Cannot write bigram probability. pos: %d, probability: %d", *entryWritingPos,
bigramEntryToWrite->getProbability());
return false;
}
}
const int targetTerminalIdToWrite =
(bigramEntryToWrite->getTargetTerminalId() == Ver4DictConstants::NOT_A_TERMINAL_ID) ?
@ -219,6 +219,6 @@ bool BigramDictContent::runGCBigramList(const int bigramListPos,
return true;
}
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime

37
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_dict_content.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_dict_content.h
View file

@ -21,18 +21,18 @@
* suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h
*/
#ifndef LATINIME_BACKWARD_V401_BIGRAM_DICT_CONTENT_H
#define LATINIME_BACKWARD_V401_BIGRAM_DICT_CONTENT_H
#ifndef LATINIME_BACKWARD_V402_BIGRAM_DICT_CONTENT_H
#define LATINIME_BACKWARD_V402_BIGRAM_DICT_CONTENT_H
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_entry.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/sparse_table_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_entry.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
class BigramDictContent : public SparseTableDictContent {
public:
@ -104,9 +104,22 @@ class BigramDictContent : public SparseTableDictContent {
private:
DISALLOW_COPY_AND_ASSIGN(BigramDictContent);
int createAndGetBigramFlags(const int probability, const bool hasNext) const {
return (probability & Ver4DictConstants::BIGRAM_PROBABILITY_MASK)
| (hasNext ? Ver4DictConstants::BIGRAM_HAS_NEXT_MASK : 0);
int createAndGetBigramFlags(const bool hasNext) const {
return hasNext ? Ver4DictConstants::BIGRAM_HAS_NEXT_MASK : 0;
}
int getBigramEntrySize() const {
if (mHasHistoricalInfo) {
return Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE
+ Ver4DictConstants::TIME_STAMP_FIELD_SIZE
+ Ver4DictConstants::WORD_LEVEL_FIELD_SIZE
+ Ver4DictConstants::WORD_COUNT_FIELD_SIZE
+ Ver4DictConstants::BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE;
} else {
return Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE
+ Ver4DictConstants::PROBABILITY_SIZE
+ Ver4DictConstants::BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE;
}
}
bool runGCBigramList(const int bigramListPos,
@ -116,7 +129,7 @@ class BigramDictContent : public SparseTableDictContent {
bool mHasHistoricalInfo;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_BIGRAM_DICT_CONTENT_H */
#endif /* LATINIME_BACKWARD_V402_BIGRAM_DICT_CONTENT_H */

12
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_entry.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_entry.h
View file

@ -21,16 +21,16 @@
* suggest/policyimpl/dictionary/structure/v4/content/bigram_entry.h
*/
#ifndef LATINIME_BACKWARD_V401_BIGRAM_ENTRY_H
#define LATINIME_BACKWARD_V401_BIGRAM_ENTRY_H
#ifndef LATINIME_BACKWARD_V402_BIGRAM_ENTRY_H
#define LATINIME_BACKWARD_V402_BIGRAM_ENTRY_H
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/historical_info.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
class BigramEntry {
public:
@ -104,7 +104,7 @@ class BigramEntry {
const HistoricalInfo mHistoricalInfo;
const int mTargetTerminalId;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_BIGRAM_ENTRY_H */
#endif /* LATINIME_BACKWARD_V402_BIGRAM_ENTRY_H */

10
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/dict_content.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/dict_content.h
View file

@ -21,14 +21,14 @@
* suggest/policyimpl/dictionary/structure/v4/content/dict_content.h
*/
#ifndef LATINIME_BACKWARD_V401_DICT_CONTENT_H
#define LATINIME_BACKWARD_V401_DICT_CONTENT_H
#ifndef LATINIME_BACKWARD_V402_DICT_CONTENT_H
#define LATINIME_BACKWARD_V402_DICT_CONTENT_H
#include "defines.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
class DictContent {
public:
@ -41,7 +41,7 @@ class DictContent {
private:
DISALLOW_COPY_AND_ASSIGN(DictContent);
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_DICT_CONTENT_H */
#endif /* LATINIME_BACKWARD_V402_DICT_CONTENT_H */

12
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/probability_dict_content.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.cpp
View file

@ -21,16 +21,16 @@
* suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.cpp
*/
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_entry.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/probability_entry.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
const ProbabilityEntry ProbabilityDictContent::getProbabilityEntry(const int terminalId) const {
if (terminalId < 0 || terminalId >= mSize) {
@ -166,6 +166,6 @@ bool ProbabilityDictContent::writeEntry(const ProbabilityEntry *const probabilit
return true;
}
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime

16
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/probability_dict_content.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.h
View file

@ -21,18 +21,18 @@
* suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h
*/
#ifndef LATINIME_BACKWARD_V401_PROBABILITY_DICT_CONTENT_H
#define LATINIME_BACKWARD_V401_PROBABILITY_DICT_CONTENT_H
#ifndef LATINIME_BACKWARD_V402_PROBABILITY_DICT_CONTENT_H
#define LATINIME_BACKWARD_V402_PROBABILITY_DICT_CONTENT_H
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/single_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/single_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
class ProbabilityEntry;
@ -68,7 +68,7 @@ class ProbabilityDictContent : public SingleDictContent {
bool mHasHistoricalInfo;
int mSize;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_PROBABILITY_DICT_CONTENT_H */
#endif /* LATINIME_BACKWARD_V402_PROBABILITY_DICT_CONTENT_H */

12
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/probability_entry.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_entry.h
View file

@ -21,16 +21,16 @@
* suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h
*/
#ifndef LATINIME_BACKWARD_V401_PROBABILITY_ENTRY_H
#define LATINIME_BACKWARD_V401_PROBABILITY_ENTRY_H
#ifndef LATINIME_BACKWARD_V402_PROBABILITY_ENTRY_H
#define LATINIME_BACKWARD_V402_PROBABILITY_ENTRY_H
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/historical_info.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
class ProbabilityEntry {
public:
@ -84,7 +84,7 @@ class ProbabilityEntry {
const int mProbability;
const HistoricalInfo mHistoricalInfo;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_PROBABILITY_ENTRY_H */
#endif /* LATINIME_BACKWARD_V402_PROBABILITY_ENTRY_H */

6
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/shortcut_dict_content.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/shortcut_dict_content.cpp
View file

@ -21,13 +21,13 @@
* suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp
*/
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/shortcut_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/shortcut_dict_content.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
void ShortcutDictContent::getShortcutEntryAndAdvancePosition(const int maxCodePointCount,
int *const outCodePoint, int *const outCodePointCount, int *const outProbability,
@ -194,6 +194,6 @@ int ShortcutDictContent::createAndGetShortcutFlags(const int probability,
| (hasNext ? Ver4DictConstants::SHORTCUT_HAS_NEXT_MASK : 0);
}
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime

16
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/shortcut_dict_content.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/shortcut_dict_content.h
View file

@ -21,17 +21,17 @@
* suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h
*/
#ifndef LATINIME_BACKWARD_V401_SHORTCUT_DICT_CONTENT_H
#define LATINIME_BACKWARD_V401_SHORTCUT_DICT_CONTENT_H
#ifndef LATINIME_BACKWARD_V402_SHORTCUT_DICT_CONTENT_H
#define LATINIME_BACKWARD_V402_SHORTCUT_DICT_CONTENT_H
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/sparse_table_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
class ShortcutDictContent : public SparseTableDictContent {
public:
@ -95,7 +95,7 @@ class ShortcutDictContent : public SparseTableDictContent {
int createAndGetShortcutFlags(const int probability, const bool hasNext) const;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_SHORTCUT_DICT_CONTENT_H */
#endif /* LATINIME_BACKWARD_V402_SHORTCUT_DICT_CONTENT_H */

14
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/single_dict_content.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/single_dict_content.h
View file

@ -21,19 +21,19 @@
* suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h
*/
#ifndef LATINIME_BACKWARD_V401_SINGLE_DICT_CONTENT_H
#define LATINIME_BACKWARD_V401_SINGLE_DICT_CONTENT_H
#ifndef LATINIME_BACKWARD_V402_SINGLE_DICT_CONTENT_H
#define LATINIME_BACKWARD_V402_SINGLE_DICT_CONTENT_H
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
class SingleDictContent : public DictContent {
public:
@ -80,7 +80,7 @@ class SingleDictContent : public DictContent {
BufferWithExtendableBuffer mExpandableContentBuffer;
const bool mIsValid;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_SINGLE_DICT_CONTENT_H */
#endif /* LATINIME_BACKWARD_V402_SINGLE_DICT_CONTENT_H */

6
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/sparse_table_dict_content.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.cpp
View file

@ -21,11 +21,11 @@
* suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp
*/
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/sparse_table_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
bool SparseTableDictContent::flush(const char *const dictPath,
const char *const lookupTableFileNameSuffix, const char *const addressTableFileNameSuffix,
@ -45,6 +45,6 @@ bool SparseTableDictContent::flush(const char *const dictPath,
return true;
}
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime

14
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/sparse_table_dict_content.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.h
View file

@ -21,12 +21,12 @@
* suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h
*/
#ifndef LATINIME_BACKWARD_V401_SPARSE_TABLE_DICT_CONTENT_H
#define LATINIME_BACKWARD_V401_SPARSE_TABLE_DICT_CONTENT_H
#ifndef LATINIME_BACKWARD_V402_SPARSE_TABLE_DICT_CONTENT_H
#define LATINIME_BACKWARD_V402_SPARSE_TABLE_DICT_CONTENT_H
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
@ -34,7 +34,7 @@
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
// TODO: Support multiple contents.
class SparseTableDictContent : public DictContent {
@ -116,7 +116,7 @@ class SparseTableDictContent : public DictContent {
SparseTable mAddressLookupTable;
const bool mIsValid;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_SPARSE_TABLE_DICT_CONTENT_H */
#endif /* LATINIME_BACKWARD_V402_SPARSE_TABLE_DICT_CONTENT_H */

8
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.cpp
View file

@ -21,14 +21,14 @@
* suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.cpp
*/
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
int TerminalPositionLookupTable::getTerminalPtNodePosition(const int terminalId) const {
if (terminalId < 0 || terminalId >= mSize) {
@ -106,6 +106,6 @@ bool TerminalPositionLookupTable::runGCTerminalIds(TerminalIdMap *const terminal
return true;
}
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime

14
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h
View file

@ -21,18 +21,18 @@
* suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h
*/
#ifndef LATINIME_BACKWARD_V401_TERMINAL_POSITION_LOOKUP_TABLE_H
#define LATINIME_BACKWARD_V401_TERMINAL_POSITION_LOOKUP_TABLE_H
#ifndef LATINIME_BACKWARD_V402_TERMINAL_POSITION_LOOKUP_TABLE_H
#define LATINIME_BACKWARD_V402_TERMINAL_POSITION_LOOKUP_TABLE_H
#include <unordered_map>
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/single_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/single_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
class TerminalPositionLookupTable : public SingleDictContent {
public:
@ -67,7 +67,7 @@ class TerminalPositionLookupTable : public SingleDictContent {
int mSize;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif // LATINIME_BACKWARD_V401_TERMINAL_POSITION_LOOKUP_TABLE_H
#endif // LATINIME_BACKWARD_V402_TERMINAL_POSITION_LOOKUP_TABLE_H

14
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/shortcut/ver4_shortcut_list_policy.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/shortcut/ver4_shortcut_list_policy.h
View file

@ -22,18 +22,18 @@
* suggest/policyimpl/dictionary/structure/v4/shortcut/ver4_shortcut_list_policy.h
*/
#ifndef LATINIME_BACKWARD_V401_VER4_SHORTCUT_LIST_POLICY_H
#define LATINIME_BACKWARD_V401_VER4_SHORTCUT_LIST_POLICY_H
#ifndef LATINIME_BACKWARD_V402_VER4_SHORTCUT_LIST_POLICY_H
#define LATINIME_BACKWARD_V402_VER4_SHORTCUT_LIST_POLICY_H
#include "defines.h"
#include "suggest/core/policy/dictionary_shortcuts_structure_policy.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/shortcut/shortcut_list_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/shortcut_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/shortcut_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
class Ver4ShortcutListPolicy : public DictionaryShortcutsStructurePolicy {
public:
@ -112,7 +112,7 @@ class Ver4ShortcutListPolicy : public DictionaryShortcutsStructurePolicy {
ShortcutDictContent *const mShortcutDictContent;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif // LATINIME_BACKWARD_V401_VER4_SHORTCUT_LIST_POLICY_H
#endif // LATINIME_BACKWARD_V402_VER4_SHORTCUT_LIST_POLICY_H

7
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.cpp
View file

@ -21,7 +21,7 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp
*/
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h"
#include <cerrno>
#include <cstring>
@ -33,7 +33,7 @@
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
/* static */ Ver4DictBuffers::Ver4DictBuffersPtr Ver4DictBuffers::openVer4DictBuffers(
const char *const dictPath, MmappedBuffer::MmappedBufferPtr headerBuffer,
@ -65,6 +65,7 @@ bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath,
return false;
}
}
umask(S_IWGRP | S_IWOTH);
if (mkdir(tmpDirPath, S_IRWXU) == -1) {
AKLOGE("Cannot create directory: %s. errno: %d.", tmpDirPath, errno);
return false;
@ -150,6 +151,6 @@ Ver4DictBuffers::Ver4DictBuffers(const HeaderPolicy *const headerPolicy, const i
mBigramDictContent(headerPolicy->hasHistoricalInfoOfWords()), mShortcutDictContent(),
mIsUpdatable(true) {}
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime

20
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h
View file

@ -21,24 +21,24 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h
*/
#ifndef LATINIME_BACKWARD_V401_VER4_DICT_BUFFER_H
#define LATINIME_BACKWARD_V401_VER4_DICT_BUFFER_H
#ifndef LATINIME_BACKWARD_V402_VER4_DICT_BUFFER_H
#define LATINIME_BACKWARD_V402_VER4_DICT_BUFFER_H
#include <memory>
#include "defines.h"
#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/shortcut_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/shortcut_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
class Ver4DictBuffers {
public:
@ -146,7 +146,7 @@ class Ver4DictBuffers {
ShortcutDictContent mShortcutDictContent;
const int mIsUpdatable;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_VER4_DICT_BUFFER_H */
#endif /* LATINIME_BACKWARD_V402_VER4_DICT_BUFFER_H */

6
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.cpp
View file

@ -21,11 +21,11 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp
*/
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
// These values MUST match the definitions in FormatSpec.java.
const char *const Ver4DictConstants::TRIE_FILE_EXTENSION = ".trie";
@ -76,6 +76,6 @@ const int Ver4DictConstants::SHORTCUT_FLAGS_FIELD_SIZE = 1;
const int Ver4DictConstants::SHORTCUT_PROBABILITY_MASK = 0x0F;
const int Ver4DictConstants::SHORTCUT_HAS_NEXT_MASK = 0x80;
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime

10
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h
View file

@ -21,14 +21,14 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h
*/
#ifndef LATINIME_BACKWARD_V401_VER4_DICT_CONSTANTS_H
#define LATINIME_BACKWARD_V401_VER4_DICT_CONSTANTS_H
#ifndef LATINIME_BACKWARD_V402_VER4_DICT_CONSTANTS_H
#define LATINIME_BACKWARD_V402_VER4_DICT_CONSTANTS_H
#include "defines.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
// TODO: Create PtConstants under the pt_common and move some constant values there.
// Note that there are corresponding definitions in FormatSpec.java.
@ -78,7 +78,7 @@ class Ver4DictConstants {
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4DictConstants);
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_VER4_DICT_CONSTANTS_H */
#endif /* LATINIME_BACKWARD_V402_VER4_DICT_CONSTANTS_H */

12
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.cpp
View file

@ -21,19 +21,19 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp
*/
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_entry.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/probability_entry.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
const PtNodeParams Ver4PatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProcessMovedPtNode(
const int ptNodePos, const int siblingNodePos) const {
@ -104,6 +104,6 @@ const PtNodeParams Ver4PatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProce
}
}
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime

18
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.h
View file

@ -21,8 +21,8 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h
*/
#ifndef LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_NODE_READER_H
#define LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_NODE_READER_H
#ifndef LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_NODE_READER_H
#define LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_NODE_READER_H
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h"
@ -30,18 +30,18 @@
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
} // namespace v401
} // namespace v402
} // namespace backward
class BufferWithExtendableBuffer;
namespace backward {
namespace v401 {
} // namespace v401
namespace v402 {
} // namespace v402
} // namespace backward
class HeaderPolicy;
namespace backward {
namespace v401 {
namespace v402 {
class ProbabilityDictContent;
/*
@ -73,7 +73,7 @@ class Ver4PatriciaTrieNodeReader : public PtNodeReader {
const PtNodeParams fetchPtNodeInfoFromBufferAndProcessMovedPtNode(const int ptNodePos,
const int siblingNodePos) const;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_NODE_READER_H */
#endif /* LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_NODE_READER_H */

16
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_writer.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_writer.cpp
View file

@ -21,24 +21,24 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp
*/
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_writer.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_writer.h"
#include "suggest/core/dictionary/property/unigram_property.h"
#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/bigram/ver4_bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_entry.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/shortcut/ver4_shortcut_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/bigram/ver4_bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/probability_entry.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/shortcut/ver4_shortcut_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
const int Ver4PatriciaTrieNodeWriter::CHILDREN_POSITION_FIELD_SIZE = 3;
@ -424,6 +424,6 @@ bool Ver4PatriciaTrieNodeWriter::updatePtNodeFlags(const int ptNodePos,
return true;
}
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime

20
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_writer.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_writer.h
View file

@ -21,29 +21,29 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h
*/
#ifndef LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_NODE_WRITER_H
#define LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_NODE_WRITER_H
#ifndef LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_NODE_WRITER_H
#define LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_NODE_WRITER_H
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_entry.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/probability_entry.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
} // namespace v401
} // namespace v402
} // namespace backward
class BufferWithExtendableBuffer;
namespace backward {
namespace v401 {
} // namespace v401
namespace v402 {
} // namespace v402
} // namespace backward
class HeaderPolicy;
namespace backward {
namespace v401 {
namespace v402 {
class Ver4BigramListPolicy;
class Ver4DictBuffers;
class Ver4PatriciaTrieNodeReader;
@ -139,7 +139,7 @@ class Ver4PatriciaTrieNodeWriter : public PtNodeWriter {
Ver4BigramListPolicy *const mBigramPolicy;
Ver4ShortcutListPolicy *const mShortcutPolicy;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_NODE_WRITER_H */
#endif /* LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_NODE_WRITER_H */

92
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_policy.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.cpp
View file

@ -22,7 +22,7 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
*/
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h"
#include <vector>
@ -33,13 +33,13 @@
#include "suggest/core/dictionary/property/word_property.h"
#include "suggest/core/session/prev_words_info.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
#include "suggest/policyimpl/dictionary/utils/probability_utils.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
// Note that there are corresponding definitions in Java side in BinaryDictionaryTests and
// BinaryDictionaryDecayingTests.
@ -70,13 +70,17 @@ void Ver4PatriciaTriePolicy::createAndGetAllChildDicNodes(const DicNode *const d
// valid terminal DicNode.
isTerminal = ptNodeParams.getProbability() != NOT_A_PROBABILITY;
}
readingHelper.readNextSiblingNode(ptNodeParams);
if (ptNodeParams.representsNonWordInfo()) {
// Skip PtNodes that represent non-word information.
continue;
}
childDicNodes->pushLeavingChild(dicNode, ptNodeParams.getHeadPos(),
ptNodeParams.getChildrenPos(), ptNodeParams.getProbability(), isTerminal,
ptNodeParams.hasChildren(),
ptNodeParams.isBlacklisted()
|| ptNodeParams.isNotAWord() /* isBlacklistedOrNotAWord */,
ptNodeParams.getCodePointCount(), ptNodeParams.getCodePoints());
readingHelper.readNextSiblingNode(ptNodeParams);
}
if (readingHelper.isError()) {
mIsCorrupted = true;
@ -122,9 +126,7 @@ int Ver4PatriciaTriePolicy::getProbability(const int unigramProbability,
} else if (bigramProbability == NOT_A_PROBABILITY) {
return ProbabilityUtils::backoff(unigramProbability);
} else {
// bigramProbability is a bigram probability delta.
return ProbabilityUtils::computeProbabilityForBigram(unigramProbability,
bigramProbability);
return bigramProbability;
}
}
}
@ -189,9 +191,19 @@ bool Ver4PatriciaTriePolicy::addUnigramEntry(const int *const word, const int le
DynamicPtReadingHelper readingHelper(&mNodeReader, &mPtNodeArrayReader);
readingHelper.initWithPtNodeArrayPos(getRootPosition());
bool addedNewUnigram = false;
if (mUpdatingHelper.addUnigramWord(&readingHelper, word, length,
int codePointsToAdd[MAX_WORD_LENGTH];
int codePointCountToAdd = length;
memmove(codePointsToAdd, word, sizeof(int) * length);
if (unigramProperty->representsBeginningOfSentence()) {
codePointCountToAdd = CharUtils::attachBeginningOfSentenceMarker(codePointsToAdd,
codePointCountToAdd, MAX_WORD_LENGTH);
}
if (codePointCountToAdd <= 0) {
return false;
}
if (mUpdatingHelper.addUnigramWord(&readingHelper, codePointsToAdd, codePointCountToAdd,
unigramProperty, &addedNewUnigram)) {
if (addedNewUnigram) {
if (addedNewUnigram && !unigramProperty->representsBeginningOfSentence()) {
mUnigramCount++;
}
if (unigramProperty->getShortcuts().size() > 0) {
@ -221,8 +233,6 @@ bool Ver4PatriciaTriePolicy::addUnigramEntry(const int *const word, const int le
bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsInfo,
const BigramProperty *const bigramProperty) {
const int length0 = prevWordsInfo->getNthPrevWordCodePointCount(1);
const int *word0 = prevWordsInfo->getNthPrevWordCodePoints(1);
if (!mBuffers->isUpdatable()) {
AKLOGI("Warning: addNgramEntry() is called for non-updatable dictionary.");
return false;
@ -232,15 +242,20 @@ bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsI
mDictBuffer->getTailPosition());
return false;
}
if (length0 > MAX_WORD_LENGTH
|| bigramProperty->getTargetCodePoints()->size() > MAX_WORD_LENGTH) {
AKLOGE("Either src word or target word is too long to insert the bigram to the dictionary. "
"length0: %d, length1: %d", length0, bigramProperty->getTargetCodePoints()->size());
if (!prevWordsInfo->isValid()) {
AKLOGE("prev words info is not valid for adding n-gram entry to the dictionary.");
return false;
}
const int word0Pos = getTerminalPtNodePositionOfWord(word0, length0,
false /* forceLowerCaseSearch */);
if (word0Pos == NOT_A_DICT_POS) {
if (bigramProperty->getTargetCodePoints()->size() > MAX_WORD_LENGTH) {
AKLOGE("The word is too long to insert the ngram to the dictionary. "
"length: %d", bigramProperty->getTargetCodePoints()->size());
return false;
}
int prevWordsPtNodePos[MAX_PREV_WORD_COUNT_FOR_N_GRAM];
prevWordsInfo->getPrevWordsTerminalPtNodePos(this, prevWordsPtNodePos,
false /* tryLowerCaseSearch */);
// TODO: Support N-gram.
if (prevWordsPtNodePos[0] == NOT_A_DICT_POS) {
return false;
}
const int word1Pos = getTerminalPtNodePositionOfWord(
@ -250,7 +265,8 @@ bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsI
return false;
}
bool addedNewBigram = false;
if (mUpdatingHelper.addBigramWords(word0Pos, word1Pos, bigramProperty, &addedNewBigram)) {
if (mUpdatingHelper.addBigramWords(prevWordsPtNodePos[0], word1Pos, bigramProperty,
&addedNewBigram)) {
if (addedNewBigram) {
mBigramCount++;
}
@ -261,11 +277,9 @@ bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsI
}
bool Ver4PatriciaTriePolicy::removeNgramEntry(const PrevWordsInfo *const prevWordsInfo,
const int *const word1, const int length1) {
const int length0 = prevWordsInfo->getNthPrevWordCodePointCount(1);
const int *word0 = prevWordsInfo->getNthPrevWordCodePoints(1);
const int *const word, const int length) {
if (!mBuffers->isUpdatable()) {
AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary.");
AKLOGI("Warning: removeNgramEntry() is called for non-updatable dictionary.");
return false;
}
if (mDictBuffer->getTailPosition() >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS) {
@ -273,22 +287,26 @@ bool Ver4PatriciaTriePolicy::removeNgramEntry(const PrevWordsInfo *const prevWor
mDictBuffer->getTailPosition());
return false;
}
if (length0 > MAX_WORD_LENGTH || length1 > MAX_WORD_LENGTH) {
AKLOGE("Either src word or target word is too long to remove the bigram to from the "
"dictionary. length0: %d, length1: %d", length0, length1);
if (!prevWordsInfo->isValid()) {
AKLOGE("prev words info is not valid for removing n-gram entry form the dictionary.");
return false;
}
const int word0Pos = getTerminalPtNodePositionOfWord(word0, length0,
if (length > MAX_WORD_LENGTH) {
AKLOGE("word is too long to remove n-gram entry form the dictionary. length: %d", length);
}
int prevWordsPtNodePos[MAX_PREV_WORD_COUNT_FOR_N_GRAM];
prevWordsInfo->getPrevWordsTerminalPtNodePos(this, prevWordsPtNodePos,
false /* tryLowerCaseSerch */);
// TODO: Support N-gram.
if (prevWordsPtNodePos[0] == NOT_A_DICT_POS) {
return false;
}
const int wordPos = getTerminalPtNodePositionOfWord(word, length,
false /* forceLowerCaseSearch */);
if (word0Pos == NOT_A_DICT_POS) {
if (wordPos == NOT_A_DICT_POS) {
return false;
}
const int word1Pos = getTerminalPtNodePositionOfWord(word1, length1,
false /* forceLowerCaseSearch */);
if (word1Pos == NOT_A_DICT_POS) {
return false;
}
if (mUpdatingHelper.removeBigramWords(word0Pos, word1Pos)) {
if (mUpdatingHelper.removeBigramWords(prevWordsPtNodePos[0], wordPos)) {
mBigramCount--;
return true;
} else {
@ -413,7 +431,7 @@ const WordProperty Ver4PatriciaTriePolicy::getWordProperty(const int *const code
const int probability = bigramEntry.hasHistoricalInfo() ?
ForgettingCurveUtils::decodeProbability(
bigramEntry.getHistoricalInfo(), mHeaderPolicy) :
getProbability(word1Probability, bigramEntry.getProbability());
bigramEntry.getProbability();
bigrams.emplace_back(&word1, probability,
historicalInfo->getTimeStamp(), historicalInfo->getLevel(),
historicalInfo->getCount());
@ -436,7 +454,7 @@ const WordProperty Ver4PatriciaTriePolicy::getWordProperty(const int *const code
shortcuts.emplace_back(&target, shortcutProbability);
}
}
const UnigramProperty unigramProperty(false /* representsBeginningOfSentence */,
const UnigramProperty unigramProperty(ptNodeParams.representsBeginningOfSentence(),
ptNodeParams.isNotAWord(), ptNodeParams.isBlacklisted(), ptNodeParams.getProbability(),
historicalInfo->getTimeStamp(), historicalInfo->getLevel(),
historicalInfo->getCount(), &shortcuts);
@ -479,6 +497,6 @@ int Ver4PatriciaTriePolicy::getNextWordAndNextToken(const int token, int *const
return nextToken;
}
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime

36
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_policy.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h
View file

@ -22,8 +22,8 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
*/
#ifndef LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_POLICY_H
#define LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_POLICY_H
#ifndef LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_POLICY_H
#define LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_POLICY_H
#include <vector>
@ -31,29 +31,29 @@
#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/bigram/ver4_bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/shortcut/ver4_shortcut_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_writer.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_writing_helper.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_pt_node_array_reader.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/bigram/ver4_bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/shortcut/ver4_shortcut_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_writer.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_writing_helper.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_pt_node_array_reader.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
} // namespace v401
} // namespace v402
} // namespace backward
class DicNode;
namespace backward {
namespace v401 {
} // namespace v401
namespace v402 {
} // namespace v402
} // namespace backward
class DicNodeVector;
namespace backward {
namespace v401 {
namespace v402 {
class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
public:
@ -114,8 +114,8 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
bool addNgramEntry(const PrevWordsInfo *const prevWordsInfo,
const BigramProperty *const bigramProperty);
bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, const int *const word,
const int length);
bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, const int *const word1,
const int length1);
bool flush(const char *const filePath);
@ -162,7 +162,7 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
std::vector<int> mTerminalPtNodePositionsForIteratingWords;
mutable bool mIsCorrupted;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif // LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_POLICY_H
#endif // LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_POLICY_H

6
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_reading_utils.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_reading_utils.cpp
View file

@ -21,19 +21,19 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.cpp
*/
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
/* static */ int Ver4PatriciaTrieReadingUtils::getTerminalIdAndAdvancePosition(
const uint8_t *const buffer, int *pos) {
return ByteArrayUtils::readUint32AndAdvancePosition(buffer, pos);
}
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime

14
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_reading_utils.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_reading_utils.h
View file

@ -21,8 +21,8 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.h
*/
#ifndef LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_READING_UTILS_H
#define LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_READING_UTILS_H
#ifndef LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_READING_UTILS_H
#define LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_READING_UTILS_H
#include <cstdint>
@ -30,13 +30,13 @@
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
} // namespace v401
} // namespace v402
} // namespace backward
class BufferWithExtendableBuffer;
namespace backward {
namespace v401 {
namespace v402 {
class Ver4PatriciaTrieReadingUtils {
public:
@ -46,7 +46,7 @@ class Ver4PatriciaTrieReadingUtils {
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4PatriciaTrieReadingUtils);
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_READING_UTILS_H */
#endif /* LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_READING_UTILS_H */

25
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_writing_helper.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_writing_helper.cpp
View file

@ -21,26 +21,26 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp
*/
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_writing_helper.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_writing_helper.h"
#include <cstring>
#include <queue>
#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/bigram/ver4_bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/shortcut/ver4_shortcut_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_writer.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_pt_node_array_reader.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/bigram/ver4_bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/shortcut/ver4_shortcut_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_writer.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_pt_node_array_reader.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/file_utils.h"
#include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
bool Ver4PatriciaTrieWritingHelper::writeToDictFile(const char *const dictDirPath,
const int unigramCount, const int bigramCount) const {
@ -222,13 +222,16 @@ bool Ver4PatriciaTrieWritingHelper::truncateUnigrams(
// Delete unigrams.
while (static_cast<int>(priorityQueue.size()) > maxUnigramCount) {
const int ptNodePos = priorityQueue.top().getDictPos();
priorityQueue.pop();
const PtNodeParams ptNodeParams =
ptNodeReader->fetchNodeInfoInBufferFromPtNodePos(ptNodePos);
if (ptNodeParams.representsNonWordInfo()) {
continue;
}
if (!ptNodeWriter->markPtNodeAsWillBecomeNonTerminal(&ptNodeParams)) {
AKLOGE("Cannot mark PtNode as willBecomeNonterminal. PtNode pos: %d", ptNodePos);
return false;
}
priorityQueue.pop();
}
return true;
}
@ -296,6 +299,6 @@ bool Ver4PatriciaTrieWritingHelper::TraversePolicyToUpdateAllPtNodeFlagsAndTermi
return mPtNodeWriter->updatePtNodeHasBigramsAndShortcutTargetsFlags(ptNodeParams);
}
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime

16
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_writing_helper.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_writing_helper.h
View file

@ -21,22 +21,22 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h
*/
#ifndef LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_WRITING_HELPER_H
#define LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_WRITING_HELPER_H
#ifndef LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_WRITING_HELPER_H
#define LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_WRITING_HELPER_H
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
} // namespace v401
} // namespace v402
} // namespace backward
class HeaderPolicy;
namespace backward {
namespace v401 {
namespace v402 {
class Ver4DictBuffers;
class Ver4PatriciaTrieNodeReader;
class Ver4PatriciaTrieNodeWriter;
@ -133,8 +133,8 @@ class Ver4PatriciaTrieWritingHelper {
Ver4DictBuffers *const mBuffers;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_WRITING_HELPER_H */
#endif /* LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_WRITING_HELPER_H */

6
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_pt_node_array_reader.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_pt_node_array_reader.cpp
View file

@ -21,7 +21,7 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_pt_node_array_reader.cpp
*/
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_pt_node_array_reader.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_pt_node_array_reader.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/patricia_trie_reading_utils.h"
@ -29,7 +29,7 @@
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
bool Ver4PtNodeArrayReader::readPtNodeArrayInfoAndReturnIfValid(const int ptNodeArrayPos,
int *const outPtNodeCount, int *const outFirstPtNodePos) const {
@ -85,6 +85,6 @@ bool Ver4PtNodeArrayReader::readForwardLinkAndReturnIfValid(const int forwordLin
return true;
}
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime

14
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_pt_node_array_reader.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_pt_node_array_reader.h
View file

@ -21,21 +21,21 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_pt_node_array_reader.h
*/
#ifndef LATINIME_BACKWARD_V401_VER4_PT_NODE_ARRAY_READER_H
#define LATINIME_BACKWARD_V401_VER4_PT_NODE_ARRAY_READER_H
#ifndef LATINIME_BACKWARD_V402_VER4_PT_NODE_ARRAY_READER_H
#define LATINIME_BACKWARD_V402_VER4_PT_NODE_ARRAY_READER_H
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_array_reader.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
} // namespace v401
} // namespace v402
} // namespace backward
class BufferWithExtendableBuffer;
namespace backward {
namespace v401 {
namespace v402 {
class Ver4PtNodeArrayReader : public PtNodeArrayReader {
public:
@ -51,7 +51,7 @@ class Ver4PtNodeArrayReader : public PtNodeArrayReader {
const BufferWithExtendableBuffer *const mBuffer;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_VER4_PT_NODE_ARRAY_READER_H */
#endif /* LATINIME_BACKWARD_V402_VER4_PT_NODE_ARRAY_READER_H */

29
native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp
View file

@ -19,9 +19,9 @@
#include <climits>
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h"
#include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h"
@ -57,14 +57,13 @@ namespace latinime {
const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap) {
FormatUtils::FORMAT_VERSION dictFormatVersion = FormatUtils::getFormatVersion(formatVersion);
switch (dictFormatVersion) {
case FormatUtils::VERSION_401: {
return newPolicyForOnMemoryV4Dict<backward::v401::Ver4DictConstants,
backward::v401::Ver4DictBuffers,
backward::v401::Ver4DictBuffers::Ver4DictBuffersPtr,
backward::v401::Ver4PatriciaTriePolicy>(
case FormatUtils::VERSION_4: {
return newPolicyForOnMemoryV4Dict<backward::v402::Ver4DictConstants,
backward::v402::Ver4DictBuffers,
backward::v402::Ver4DictBuffers::Ver4DictBuffersPtr,
backward::v402::Ver4PatriciaTriePolicy>(
dictFormatVersion, locale, attributeMap);
}
case FormatUtils::VERSION_4:
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
case FormatUtils::VERSION_4_DEV: {
return newPolicyForOnMemoryV4Dict<Ver4DictConstants, Ver4DictBuffers,
@ -116,14 +115,13 @@ template<class DictConstants, class DictBuffers, class DictBuffersPtr, class Str
case FormatUtils::VERSION_2:
AKLOGE("Given path is a directory but the format is version 2. path: %s", path);
break;
case FormatUtils::VERSION_401: {
return newPolicyForV4Dict<backward::v401::Ver4DictConstants,
backward::v401::Ver4DictBuffers,
backward::v401::Ver4DictBuffers::Ver4DictBuffersPtr,
backward::v401::Ver4PatriciaTriePolicy>(
case FormatUtils::VERSION_4: {
return newPolicyForV4Dict<backward::v402::Ver4DictConstants,
backward::v402::Ver4DictBuffers,
backward::v402::Ver4DictBuffers::Ver4DictBuffersPtr,
backward::v402::Ver4PatriciaTriePolicy>(
headerFilePath, formatVersion, std::move(mmappedBuffer));
}
case FormatUtils::VERSION_4:
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
case FormatUtils::VERSION_4_DEV: {
return newPolicyForV4Dict<Ver4DictConstants, Ver4DictBuffers,
@ -179,7 +177,6 @@ template<class DictConstants, class DictBuffers, class DictBuffersPtr, class Str
case FormatUtils::VERSION_2:
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(
new PatriciaTriePolicy(std::move(mmappedBuffer)));
case FormatUtils::VERSION_401:
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
case FormatUtils::VERSION_4:
case FormatUtils::VERSION_4_DEV:

11
native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
View file

@ -23,7 +23,7 @@
#include <sys/types.h>
#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
@ -41,12 +41,11 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE =
TimeKeeper::setCurrentTime();
const FormatUtils::FORMAT_VERSION formatVersion = FormatUtils::getFormatVersion(dictVersion);
switch (formatVersion) {
case FormatUtils::VERSION_401:
return createEmptyV4DictFile<backward::v401::Ver4DictConstants,
backward::v401::Ver4DictBuffers,
backward::v401::Ver4DictBuffers::Ver4DictBuffersPtr>(
filePath, localeAsCodePointVector, attributeMap, formatVersion);
case FormatUtils::VERSION_4:
return createEmptyV4DictFile<backward::v402::Ver4DictConstants,
backward::v402::Ver4DictBuffers,
backward::v402::Ver4DictBuffers::Ver4DictBuffersPtr>(
filePath, localeAsCodePointVector, attributeMap, formatVersion);
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
case FormatUtils::VERSION_4_DEV:
return createEmptyV4DictFile<Ver4DictConstants, Ver4DictBuffers,

18
native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp
View file

@ -29,8 +29,6 @@ const int FormatUtils::DICTIONARY_MINIMUM_SIZE = 12;
switch (formatVersion) {
case VERSION_2:
return VERSION_2;
case VERSION_401:
return VERSION_401;
case VERSION_4_ONLY_FOR_TESTING:
return VERSION_4_ONLY_FOR_TESTING;
case VERSION_4:
@ -52,7 +50,7 @@ const int FormatUtils::DICTIONARY_MINIMUM_SIZE = 12;
const uint32_t magicNumber = ByteArrayUtils::readUint32(dict, 0);
switch (magicNumber) {
case MAGIC_NUMBER:
// Version 2 header is as follows:
// The layout of the header is as follows:
// Magic number (4 bytes) 0x9B 0xC1 0x3A 0xFE
// Dictionary format version number (2 bytes)
// Options (2 bytes)
@ -60,19 +58,7 @@ const int FormatUtils::DICTIONARY_MINIMUM_SIZE = 12;
// Conceptually this converts the hardcoded value of the bytes in the file into
// the symbolic value we use in the code. But we want the constants to be the
// same so we use them for both here.
if (ByteArrayUtils::readUint16(dict, 4) == VERSION_2) {
return VERSION_2;
} else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_401) {
return VERSION_401;
} else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_4_ONLY_FOR_TESTING) {
return VERSION_4_ONLY_FOR_TESTING;
} else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_4) {
return VERSION_4;
} else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_4_DEV) {
return VERSION_4_DEV;
} else {
return UNKNOWN_VERSION;
}
return getFormatVersion(ByteArrayUtils::readUint16(dict, 4));
default:
return UNKNOWN_VERSION;
}

1
native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h
View file

@ -32,7 +32,6 @@ class FormatUtils {
// These MUST have the same values as the relevant constants in FormatSpec.java.
VERSION_2 = 2,
VERSION_4_ONLY_FOR_TESTING = 399,
VERSION_401 = 401,
VERSION_4 = 402,
VERSION_4_DEV = 403,
UNKNOWN_VERSION = -1