nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Purge dict reading code for 401 and create for 402. 

Change-Id: Ib0150b3306bb2db8aff443d1158fecc8464cbbdb
main
Keisuke Kuroyanagi 2014-06-10 19:22:40 +09:00
parent 2385314be0
commit 07e1412631
42 changed files with 333 additions and 321 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
View File

@ -114,8 +114,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
private boolean needsToMigrateDictionary(final int formatVersion) {
// When we bump up the dictionary format version, the old version should be added to here
// for supporting migration. Note that native code has to support reading such formats.
return formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING
|| formatVersion == FormatSpec.VERSION401;
return formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING;
}
public boolean isValidDictionaryLocked() {

10
native/jni/NativeFileList.mk
View File

@ -100,8 +100,8 @@ LATIN_IME_CORE_SRC_FILES := \
log_utils.cpp \
time_keeper.cpp)
LATIN_IME_CORE_SRC_FILES_BACKWARD_V401 := \
$(addprefix suggest/policyimpl/dictionary/structure/backward/v401/, \
LATIN_IME_CORE_SRC_FILES_BACKWARD_V402 := \
$(addprefix suggest/policyimpl/dictionary/structure/backward/v402/, \
ver4_dict_buffers.cpp \
ver4_dict_constants.cpp \
ver4_patricia_trie_node_reader.cpp \
@ -110,16 +110,16 @@ LATIN_IME_CORE_SRC_FILES_BACKWARD_V401 := \
ver4_patricia_trie_reading_utils.cpp \
ver4_patricia_trie_writing_helper.cpp \
ver4_pt_node_array_reader.cpp) \
$(addprefix suggest/policyimpl/dictionary/structure/backward/v401/content/, \
$(addprefix suggest/policyimpl/dictionary/structure/backward/v402/content/, \
bigram_dict_content.cpp \
probability_dict_content.cpp \
shortcut_dict_content.cpp \
sparse_table_dict_content.cpp \
terminal_position_lookup_table.cpp) \
$(addprefix suggest/policyimpl/dictionary/structure/backward/v401/bigram/, \
$(addprefix suggest/policyimpl/dictionary/structure/backward/v402/bigram/, \
ver4_bigram_list_policy.cpp)
LATIN_IME_CORE_SRC_FILES += $(LATIN_IME_CORE_SRC_FILES_BACKWARD_V401)
LATIN_IME_CORE_SRC_FILES += $(LATIN_IME_CORE_SRC_FILES_BACKWARD_V402)
LATIN_IME_CORE_TEST_FILES := \
defines_test.cpp \

4
native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
View File

@ -139,8 +139,6 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
switch (mDictFormatVersion) {
case FormatUtils::VERSION_2:
return FormatUtils::VERSION_2;
case FormatUtils::VERSION_401:
return FormatUtils::VERSION_401;
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
return FormatUtils::VERSION_4_ONLY_FOR_TESTING;
case FormatUtils::VERSION_4:
@ -249,7 +247,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
}
bool supportsBeginningOfSentence() const {
return mDictFormatVersion > FormatUtils::VERSION_401;
return mDictFormatVersion >= FormatUtils::VERSION_4;
}
private:

1
native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp
View File

@ -98,7 +98,6 @@ typedef DictionaryHeaderStructurePolicy::AttributeMap AttributeMap;
case FormatUtils::VERSION_2:
// Version 2 dictionary writing is not supported.
return false;
case FormatUtils::VERSION_401:
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
case FormatUtils::VERSION_4:
case FormatUtils::VERSION_4_DEV:

0
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/Readme.txt → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/Readme.txt
View File

12
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/bigram/ver4_bigram_list_policy.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/bigram/ver4_bigram_list_policy.cpp
View File

@ -22,19 +22,19 @@
* suggest/policyimpl/dictionary/structure/v4/bigram/ver4_bigram_list_policy.cpp
*/
#include "suggest/policyimpl/dictionary/structure/backward/v401/bigram/ver4_bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/bigram/ver4_bigram_list_policy.h"
#include "suggest/core/dictionary/property/bigram_property.h"
#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/bigram/bigram_list_read_write_utils.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
void Ver4BigramListPolicy::getNextBigram(int *const outBigramPos, int *const outProbability,
bool *const outHasNext, int *const bigramEntryPos) const {
@ -285,6 +285,6 @@ bool Ver4BigramListPolicy::updateHasNextFlag(const bool hasNext, const int bigra
return mBigramDictContent->writeBigramEntry(&updatedBigramEntry, bigramEntryPos);
}
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime

20
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/bigram/ver4_bigram_list_policy.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/bigram/ver4_bigram_list_policy.h
View File

@ -22,28 +22,28 @@
* suggest/policyimpl/dictionary/structure/v4/bigram/ver4_bigram_list_policy.h
*/
#ifndef LATINIME_BACKWARD_V401_VER4_BIGRAM_LIST_POLICY_H
#define LATINIME_BACKWARD_V401_VER4_BIGRAM_LIST_POLICY_H
#ifndef LATINIME_BACKWARD_V402_VER4_BIGRAM_LIST_POLICY_H
#define LATINIME_BACKWARD_V402_VER4_BIGRAM_LIST_POLICY_H
#include "defines.h"
#include "suggest/core/policy/dictionary_bigrams_structure_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_entry.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_entry.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
class BigramDictContent;
} // namespace v401
} // namespace v402
} // namespace backward
class BigramProperty;
namespace backward {
namespace v401 {
} // namespace v401
namespace v402 {
} // namespace v402
} // namespace backward
class HeaderPolicy;
namespace backward {
namespace v401 {
namespace v402 {
class TerminalPositionLookupTable;
class Ver4BigramListPolicy : public DictionaryBigramsStructurePolicy {
@ -87,7 +87,7 @@ class Ver4BigramListPolicy : public DictionaryBigramsStructurePolicy {
const TerminalPositionLookupTable *const mTerminalPositionLookupTable;
const HeaderPolicy *const mHeaderPolicy;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_VER4_BIGRAM_LIST_POLICY_H */
#endif /* LATINIME_BACKWARD_V402_VER4_BIGRAM_LIST_POLICY_H */

36
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_dict_content.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_dict_content.cpp
View File

@ -21,20 +21,22 @@
* suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.cpp
*/
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_dict_content.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition(
int *const bigramEntryPos) const {
const BufferWithExtendableBuffer *const bigramListBuffer = getContentBuffer();
if (*bigramEntryPos < 0 || *bigramEntryPos >= bigramListBuffer->getTailPosition()) {
AKLOGE("Invalid bigram entry position. bigramEntryPos: %d, bufSize: %d",
*bigramEntryPos, bigramListBuffer->getTailPosition());
const int bigramEntryTailPos = (*bigramEntryPos) + getBigramEntrySize();
if (*bigramEntryPos < 0 || bigramEntryTailPos > bigramListBuffer->getTailPosition()) {
AKLOGE("Invalid bigram entry position. bigramEntryPos: %d, bigramEntryTailPos: %d, "
"bufSize: %d", *bigramEntryPos, bigramEntryTailPos,
bigramListBuffer->getTailPosition());
ASSERT(false);
return BigramEntry(false /* hasNext */, NOT_A_PROBABILITY,
Ver4DictConstants::NOT_A_TERMINAL_ID);
@ -47,8 +49,6 @@ const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition(
int level = 0;
int count = 0;
if (mHasHistoricalInfo) {
probability = bigramListBuffer->readUintAndAdvancePosition(
Ver4DictConstants::PROBABILITY_SIZE, bigramEntryPos);
timestamp = bigramListBuffer->readUintAndAdvancePosition(
Ver4DictConstants::TIME_STAMP_FIELD_SIZE, bigramEntryPos);
level = bigramListBuffer->readUintAndAdvancePosition(
@ -56,7 +56,8 @@ const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition(
count = bigramListBuffer->readUintAndAdvancePosition(
Ver4DictConstants::WORD_COUNT_FIELD_SIZE, bigramEntryPos);
} else {
probability = bigramFlags & Ver4DictConstants::BIGRAM_PROBABILITY_MASK;
probability = bigramListBuffer->readUintAndAdvancePosition(
Ver4DictConstants::PROBABILITY_SIZE, bigramEntryPos);
}
const int encodedTargetTerminalId = bigramListBuffer->readUintAndAdvancePosition(
Ver4DictConstants::BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE, bigramEntryPos);
@ -74,21 +75,13 @@ const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition(
bool BigramDictContent::writeBigramEntryAndAdvancePosition(
const BigramEntry *const bigramEntryToWrite, int *const entryWritingPos) {
BufferWithExtendableBuffer *const bigramListBuffer = getWritableContentBuffer();
const int bigramFlags = createAndGetBigramFlags(
mHasHistoricalInfo ? 0 : bigramEntryToWrite->getProbability(),
bigramEntryToWrite->hasNext());
const int bigramFlags = createAndGetBigramFlags(bigramEntryToWrite->hasNext());
if (!bigramListBuffer->writeUintAndAdvancePosition(bigramFlags,
Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE, entryWritingPos)) {
AKLOGE("Cannot write bigram flags. pos: %d, flags: %x", *entryWritingPos, bigramFlags);
return false;
}
if (mHasHistoricalInfo) {
if (!bigramListBuffer->writeUintAndAdvancePosition(bigramEntryToWrite->getProbability(),
Ver4DictConstants::PROBABILITY_SIZE, entryWritingPos)) {
AKLOGE("Cannot write bigram probability. pos: %d, probability: %d", *entryWritingPos,
bigramEntryToWrite->getProbability());
return false;
}
const HistoricalInfo *const historicalInfo = bigramEntryToWrite->getHistoricalInfo();
if (!bigramListBuffer->writeUintAndAdvancePosition(historicalInfo->getTimeStamp(),
Ver4DictConstants::TIME_STAMP_FIELD_SIZE, entryWritingPos)) {
@ -108,6 +101,13 @@ bool BigramDictContent::writeBigramEntryAndAdvancePosition(
historicalInfo->getCount());
return false;
}
} else {
if (!bigramListBuffer->writeUintAndAdvancePosition(bigramEntryToWrite->getProbability(),
Ver4DictConstants::PROBABILITY_SIZE, entryWritingPos)) {
AKLOGE("Cannot write bigram probability. pos: %d, probability: %d", *entryWritingPos,
bigramEntryToWrite->getProbability());
return false;
}
}
const int targetTerminalIdToWrite =
(bigramEntryToWrite->getTargetTerminalId() == Ver4DictConstants::NOT_A_TERMINAL_ID) ?
@ -219,6 +219,6 @@ bool BigramDictContent::runGCBigramList(const int bigramListPos,
return true;
}
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime

37
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_dict_content.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_dict_content.h
View File

@ -21,18 +21,18 @@
* suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h
*/
#ifndef LATINIME_BACKWARD_V401_BIGRAM_DICT_CONTENT_H
#define LATINIME_BACKWARD_V401_BIGRAM_DICT_CONTENT_H
#ifndef LATINIME_BACKWARD_V402_BIGRAM_DICT_CONTENT_H
#define LATINIME_BACKWARD_V402_BIGRAM_DICT_CONTENT_H
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_entry.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/sparse_table_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_entry.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
class BigramDictContent : public SparseTableDictContent {
public:
@ -104,9 +104,22 @@ class BigramDictContent : public SparseTableDictContent {
private:
DISALLOW_COPY_AND_ASSIGN(BigramDictContent);
int createAndGetBigramFlags(const int probability, const bool hasNext) const {
return (probability & Ver4DictConstants::BIGRAM_PROBABILITY_MASK)
| (hasNext ? Ver4DictConstants::BIGRAM_HAS_NEXT_MASK : 0);
int createAndGetBigramFlags(const bool hasNext) const {
return hasNext ? Ver4DictConstants::BIGRAM_HAS_NEXT_MASK : 0;
}
int getBigramEntrySize() const {
if (mHasHistoricalInfo) {
return Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE
+ Ver4DictConstants::TIME_STAMP_FIELD_SIZE
+ Ver4DictConstants::WORD_LEVEL_FIELD_SIZE
+ Ver4DictConstants::WORD_COUNT_FIELD_SIZE
+ Ver4DictConstants::BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE;
} else {
return Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE
+ Ver4DictConstants::PROBABILITY_SIZE
+ Ver4DictConstants::BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE;
}
}
bool runGCBigramList(const int bigramListPos,
@ -116,7 +129,7 @@ class BigramDictContent : public SparseTableDictContent {
bool mHasHistoricalInfo;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_BIGRAM_DICT_CONTENT_H */
#endif /* LATINIME_BACKWARD_V402_BIGRAM_DICT_CONTENT_H */

12
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_entry.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_entry.h
View File

@ -21,16 +21,16 @@
* suggest/policyimpl/dictionary/structure/v4/content/bigram_entry.h
*/
#ifndef LATINIME_BACKWARD_V401_BIGRAM_ENTRY_H
#define LATINIME_BACKWARD_V401_BIGRAM_ENTRY_H
#ifndef LATINIME_BACKWARD_V402_BIGRAM_ENTRY_H
#define LATINIME_BACKWARD_V402_BIGRAM_ENTRY_H
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/historical_info.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
class BigramEntry {
public:
@ -104,7 +104,7 @@ class BigramEntry {
const HistoricalInfo mHistoricalInfo;
const int mTargetTerminalId;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_BIGRAM_ENTRY_H */
#endif /* LATINIME_BACKWARD_V402_BIGRAM_ENTRY_H */

10
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/dict_content.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/dict_content.h
View File

@ -21,14 +21,14 @@
* suggest/policyimpl/dictionary/structure/v4/content/dict_content.h
*/
#ifndef LATINIME_BACKWARD_V401_DICT_CONTENT_H
#define LATINIME_BACKWARD_V401_DICT_CONTENT_H
#ifndef LATINIME_BACKWARD_V402_DICT_CONTENT_H
#define LATINIME_BACKWARD_V402_DICT_CONTENT_H
#include "defines.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
class DictContent {
public:
@ -41,7 +41,7 @@ class DictContent {
private:
DISALLOW_COPY_AND_ASSIGN(DictContent);
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_DICT_CONTENT_H */
#endif /* LATINIME_BACKWARD_V402_DICT_CONTENT_H */

12
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/probability_dict_content.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.cpp
View File

@ -21,16 +21,16 @@
* suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.cpp
*/
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_entry.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/probability_entry.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
const ProbabilityEntry ProbabilityDictContent::getProbabilityEntry(const int terminalId) const {
if (terminalId < 0 || terminalId >= mSize) {
@ -166,6 +166,6 @@ bool ProbabilityDictContent::writeEntry(const ProbabilityEntry *const probabilit
return true;
}
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime

16
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/probability_dict_content.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.h
View File

@ -21,18 +21,18 @@
* suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h
*/
#ifndef LATINIME_BACKWARD_V401_PROBABILITY_DICT_CONTENT_H
#define LATINIME_BACKWARD_V401_PROBABILITY_DICT_CONTENT_H
#ifndef LATINIME_BACKWARD_V402_PROBABILITY_DICT_CONTENT_H
#define LATINIME_BACKWARD_V402_PROBABILITY_DICT_CONTENT_H
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/single_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/single_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
class ProbabilityEntry;
@ -68,7 +68,7 @@ class ProbabilityDictContent : public SingleDictContent {
bool mHasHistoricalInfo;
int mSize;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_PROBABILITY_DICT_CONTENT_H */
#endif /* LATINIME_BACKWARD_V402_PROBABILITY_DICT_CONTENT_H */

12
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/probability_entry.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_entry.h
View File

@ -21,16 +21,16 @@
* suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h
*/
#ifndef LATINIME_BACKWARD_V401_PROBABILITY_ENTRY_H
#define LATINIME_BACKWARD_V401_PROBABILITY_ENTRY_H
#ifndef LATINIME_BACKWARD_V402_PROBABILITY_ENTRY_H
#define LATINIME_BACKWARD_V402_PROBABILITY_ENTRY_H
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/historical_info.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
class ProbabilityEntry {
public:
@ -84,7 +84,7 @@ class ProbabilityEntry {
const int mProbability;
const HistoricalInfo mHistoricalInfo;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_PROBABILITY_ENTRY_H */
#endif /* LATINIME_BACKWARD_V402_PROBABILITY_ENTRY_H */

6
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/shortcut_dict_content.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/shortcut_dict_content.cpp
View File

@ -21,13 +21,13 @@
* suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp
*/
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/shortcut_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/shortcut_dict_content.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
void ShortcutDictContent::getShortcutEntryAndAdvancePosition(const int maxCodePointCount,
int *const outCodePoint, int *const outCodePointCount, int *const outProbability,
@ -194,6 +194,6 @@ int ShortcutDictContent::createAndGetShortcutFlags(const int probability,
| (hasNext ? Ver4DictConstants::SHORTCUT_HAS_NEXT_MASK : 0);
}
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime

16
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/shortcut_dict_content.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/shortcut_dict_content.h
View File

@ -21,17 +21,17 @@
* suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h
*/
#ifndef LATINIME_BACKWARD_V401_SHORTCUT_DICT_CONTENT_H
#define LATINIME_BACKWARD_V401_SHORTCUT_DICT_CONTENT_H
#ifndef LATINIME_BACKWARD_V402_SHORTCUT_DICT_CONTENT_H
#define LATINIME_BACKWARD_V402_SHORTCUT_DICT_CONTENT_H
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/sparse_table_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
class ShortcutDictContent : public SparseTableDictContent {
public:
@ -95,7 +95,7 @@ class ShortcutDictContent : public SparseTableDictContent {
int createAndGetShortcutFlags(const int probability, const bool hasNext) const;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_SHORTCUT_DICT_CONTENT_H */
#endif /* LATINIME_BACKWARD_V402_SHORTCUT_DICT_CONTENT_H */

14
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/single_dict_content.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/single_dict_content.h
View File

@ -21,19 +21,19 @@
* suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h
*/
#ifndef LATINIME_BACKWARD_V401_SINGLE_DICT_CONTENT_H
#define LATINIME_BACKWARD_V401_SINGLE_DICT_CONTENT_H
#ifndef LATINIME_BACKWARD_V402_SINGLE_DICT_CONTENT_H
#define LATINIME_BACKWARD_V402_SINGLE_DICT_CONTENT_H
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
class SingleDictContent : public DictContent {
public:
@ -80,7 +80,7 @@ class SingleDictContent : public DictContent {
BufferWithExtendableBuffer mExpandableContentBuffer;
const bool mIsValid;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_SINGLE_DICT_CONTENT_H */
#endif /* LATINIME_BACKWARD_V402_SINGLE_DICT_CONTENT_H */

6
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/sparse_table_dict_content.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.cpp
View File

@ -21,11 +21,11 @@
* suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp
*/
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/sparse_table_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
bool SparseTableDictContent::flush(const char *const dictPath,
const char *const lookupTableFileNameSuffix, const char *const addressTableFileNameSuffix,
@ -45,6 +45,6 @@ bool SparseTableDictContent::flush(const char *const dictPath,
return true;
}
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime

14
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/sparse_table_dict_content.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.h
View File

@ -21,12 +21,12 @@
* suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h
*/
#ifndef LATINIME_BACKWARD_V401_SPARSE_TABLE_DICT_CONTENT_H
#define LATINIME_BACKWARD_V401_SPARSE_TABLE_DICT_CONTENT_H
#ifndef LATINIME_BACKWARD_V402_SPARSE_TABLE_DICT_CONTENT_H
#define LATINIME_BACKWARD_V402_SPARSE_TABLE_DICT_CONTENT_H
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
@ -34,7 +34,7 @@
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
// TODO: Support multiple contents.
class SparseTableDictContent : public DictContent {
@ -116,7 +116,7 @@ class SparseTableDictContent : public DictContent {
SparseTable mAddressLookupTable;
const bool mIsValid;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_SPARSE_TABLE_DICT_CONTENT_H */
#endif /* LATINIME_BACKWARD_V402_SPARSE_TABLE_DICT_CONTENT_H */

8
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.cpp
View File

@ -21,14 +21,14 @@
* suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.cpp
*/
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
int TerminalPositionLookupTable::getTerminalPtNodePosition(const int terminalId) const {
if (terminalId < 0 || terminalId >= mSize) {
@ -106,6 +106,6 @@ bool TerminalPositionLookupTable::runGCTerminalIds(TerminalIdMap *const terminal
return true;
}
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime

14
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h
View File

@ -21,18 +21,18 @@
* suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h
*/
#ifndef LATINIME_BACKWARD_V401_TERMINAL_POSITION_LOOKUP_TABLE_H
#define LATINIME_BACKWARD_V401_TERMINAL_POSITION_LOOKUP_TABLE_H
#ifndef LATINIME_BACKWARD_V402_TERMINAL_POSITION_LOOKUP_TABLE_H
#define LATINIME_BACKWARD_V402_TERMINAL_POSITION_LOOKUP_TABLE_H
#include <unordered_map>
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/single_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/single_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
class TerminalPositionLookupTable : public SingleDictContent {
public:
@ -67,7 +67,7 @@ class TerminalPositionLookupTable : public SingleDictContent {
int mSize;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif // LATINIME_BACKWARD_V401_TERMINAL_POSITION_LOOKUP_TABLE_H
#endif // LATINIME_BACKWARD_V402_TERMINAL_POSITION_LOOKUP_TABLE_H

14
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/shortcut/ver4_shortcut_list_policy.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/shortcut/ver4_shortcut_list_policy.h
View File

@ -22,18 +22,18 @@
* suggest/policyimpl/dictionary/structure/v4/shortcut/ver4_shortcut_list_policy.h
*/
#ifndef LATINIME_BACKWARD_V401_VER4_SHORTCUT_LIST_POLICY_H
#define LATINIME_BACKWARD_V401_VER4_SHORTCUT_LIST_POLICY_H
#ifndef LATINIME_BACKWARD_V402_VER4_SHORTCUT_LIST_POLICY_H
#define LATINIME_BACKWARD_V402_VER4_SHORTCUT_LIST_POLICY_H
#include "defines.h"
#include "suggest/core/policy/dictionary_shortcuts_structure_policy.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/shortcut/shortcut_list_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/shortcut_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/shortcut_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
class Ver4ShortcutListPolicy : public DictionaryShortcutsStructurePolicy {
public:
@ -112,7 +112,7 @@ class Ver4ShortcutListPolicy : public DictionaryShortcutsStructurePolicy {
ShortcutDictContent *const mShortcutDictContent;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif // LATINIME_BACKWARD_V401_VER4_SHORTCUT_LIST_POLICY_H
#endif // LATINIME_BACKWARD_V402_VER4_SHORTCUT_LIST_POLICY_H

7
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.cpp
View File

@ -21,7 +21,7 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp
*/
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h"
#include <cerrno>
#include <cstring>
@ -33,7 +33,7 @@
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
/* static */ Ver4DictBuffers::Ver4DictBuffersPtr Ver4DictBuffers::openVer4DictBuffers(
const char *const dictPath, MmappedBuffer::MmappedBufferPtr headerBuffer,
@ -65,6 +65,7 @@ bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath,
return false;
}
}
umask(S_IWGRP | S_IWOTH);
if (mkdir(tmpDirPath, S_IRWXU) == -1) {
AKLOGE("Cannot create directory: %s. errno: %d.", tmpDirPath, errno);
return false;
@ -150,6 +151,6 @@ Ver4DictBuffers::Ver4DictBuffers(const HeaderPolicy *const headerPolicy, const i
mBigramDictContent(headerPolicy->hasHistoricalInfoOfWords()), mShortcutDictContent(),
mIsUpdatable(true) {}
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime

20
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h
View File

@ -21,24 +21,24 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h
*/
#ifndef LATINIME_BACKWARD_V401_VER4_DICT_BUFFER_H
#define LATINIME_BACKWARD_V401_VER4_DICT_BUFFER_H
#ifndef LATINIME_BACKWARD_V402_VER4_DICT_BUFFER_H
#define LATINIME_BACKWARD_V402_VER4_DICT_BUFFER_H
#include <memory>
#include "defines.h"
#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/shortcut_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/shortcut_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
class Ver4DictBuffers {
public:
@ -146,7 +146,7 @@ class Ver4DictBuffers {
ShortcutDictContent mShortcutDictContent;
const int mIsUpdatable;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_VER4_DICT_BUFFER_H */
#endif /* LATINIME_BACKWARD_V402_VER4_DICT_BUFFER_H */

6
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.cpp
View File

@ -21,11 +21,11 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp
*/
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
// These values MUST match the definitions in FormatSpec.java.
const char *const Ver4DictConstants::TRIE_FILE_EXTENSION = ".trie";
@ -76,6 +76,6 @@ const int Ver4DictConstants::SHORTCUT_FLAGS_FIELD_SIZE = 1;
const int Ver4DictConstants::SHORTCUT_PROBABILITY_MASK = 0x0F;
const int Ver4DictConstants::SHORTCUT_HAS_NEXT_MASK = 0x80;
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime

10
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h
View File

@ -21,14 +21,14 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h
*/
#ifndef LATINIME_BACKWARD_V401_VER4_DICT_CONSTANTS_H
#define LATINIME_BACKWARD_V401_VER4_DICT_CONSTANTS_H
#ifndef LATINIME_BACKWARD_V402_VER4_DICT_CONSTANTS_H
#define LATINIME_BACKWARD_V402_VER4_DICT_CONSTANTS_H
#include "defines.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
// TODO: Create PtConstants under the pt_common and move some constant values there.
// Note that there are corresponding definitions in FormatSpec.java.
@ -78,7 +78,7 @@ class Ver4DictConstants {
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4DictConstants);
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_VER4_DICT_CONSTANTS_H */
#endif /* LATINIME_BACKWARD_V402_VER4_DICT_CONSTANTS_H */

12
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.cpp
View File

@ -21,19 +21,19 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp
*/
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_entry.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/probability_entry.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
const PtNodeParams Ver4PatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProcessMovedPtNode(
const int ptNodePos, const int siblingNodePos) const {
@ -104,6 +104,6 @@ const PtNodeParams Ver4PatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProce
}
}
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime

18
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.h
View File

@ -21,8 +21,8 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h
*/
#ifndef LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_NODE_READER_H
#define LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_NODE_READER_H
#ifndef LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_NODE_READER_H
#define LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_NODE_READER_H
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h"
@ -30,18 +30,18 @@
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
} // namespace v401
} // namespace v402
} // namespace backward
class BufferWithExtendableBuffer;
namespace backward {
namespace v401 {
} // namespace v401
namespace v402 {
} // namespace v402
} // namespace backward
class HeaderPolicy;
namespace backward {
namespace v401 {
namespace v402 {
class ProbabilityDictContent;
/*
@ -73,7 +73,7 @@ class Ver4PatriciaTrieNodeReader : public PtNodeReader {
const PtNodeParams fetchPtNodeInfoFromBufferAndProcessMovedPtNode(const int ptNodePos,
const int siblingNodePos) const;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_NODE_READER_H */
#endif /* LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_NODE_READER_H */

16
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_writer.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_writer.cpp
View File

@ -21,24 +21,24 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp
*/
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_writer.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_writer.h"
#include "suggest/core/dictionary/property/unigram_property.h"
#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/bigram/ver4_bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_entry.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/shortcut/ver4_shortcut_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/bigram/ver4_bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/probability_entry.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/shortcut/ver4_shortcut_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
const int Ver4PatriciaTrieNodeWriter::CHILDREN_POSITION_FIELD_SIZE = 3;
@ -424,6 +424,6 @@ bool Ver4PatriciaTrieNodeWriter::updatePtNodeFlags(const int ptNodePos,
return true;
}
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime

20
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_writer.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_writer.h
View File

@ -21,29 +21,29 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h
*/
#ifndef LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_NODE_WRITER_H
#define LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_NODE_WRITER_H
#ifndef LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_NODE_WRITER_H
#define LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_NODE_WRITER_H
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_entry.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/probability_entry.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
} // namespace v401
} // namespace v402
} // namespace backward
class BufferWithExtendableBuffer;
namespace backward {
namespace v401 {
} // namespace v401
namespace v402 {
} // namespace v402
} // namespace backward
class HeaderPolicy;
namespace backward {
namespace v401 {
namespace v402 {
class Ver4BigramListPolicy;
class Ver4DictBuffers;
class Ver4PatriciaTrieNodeReader;
@ -139,7 +139,7 @@ class Ver4PatriciaTrieNodeWriter : public PtNodeWriter {
Ver4BigramListPolicy *const mBigramPolicy;
Ver4ShortcutListPolicy *const mShortcutPolicy;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_NODE_WRITER_H */
#endif /* LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_NODE_WRITER_H */

92
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_policy.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.cpp
View File

@ -22,7 +22,7 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
*/
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h"
#include <vector>
@ -33,13 +33,13 @@
#include "suggest/core/dictionary/property/word_property.h"
#include "suggest/core/session/prev_words_info.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
#include "suggest/policyimpl/dictionary/utils/probability_utils.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
// Note that there are corresponding definitions in Java side in BinaryDictionaryTests and
// BinaryDictionaryDecayingTests.
@ -70,13 +70,17 @@ void Ver4PatriciaTriePolicy::createAndGetAllChildDicNodes(const DicNode *const d
// valid terminal DicNode.
isTerminal = ptNodeParams.getProbability() != NOT_A_PROBABILITY;
}
readingHelper.readNextSiblingNode(ptNodeParams);
if (ptNodeParams.representsNonWordInfo()) {
// Skip PtNodes that represent non-word information.
continue;
}
childDicNodes->pushLeavingChild(dicNode, ptNodeParams.getHeadPos(),
ptNodeParams.getChildrenPos(), ptNodeParams.getProbability(), isTerminal,
ptNodeParams.hasChildren(),
ptNodeParams.isBlacklisted()
|| ptNodeParams.isNotAWord() /* isBlacklistedOrNotAWord */,
ptNodeParams.getCodePointCount(), ptNodeParams.getCodePoints());
readingHelper.readNextSiblingNode(ptNodeParams);
}
if (readingHelper.isError()) {
mIsCorrupted = true;
@ -122,9 +126,7 @@ int Ver4PatriciaTriePolicy::getProbability(const int unigramProbability,
} else if (bigramProbability == NOT_A_PROBABILITY) {
return ProbabilityUtils::backoff(unigramProbability);
} else {
// bigramProbability is a bigram probability delta.
return ProbabilityUtils::computeProbabilityForBigram(unigramProbability,
bigramProbability);
return bigramProbability;
}
}
}
@ -189,9 +191,19 @@ bool Ver4PatriciaTriePolicy::addUnigramEntry(const int *const word, const int le
DynamicPtReadingHelper readingHelper(&mNodeReader, &mPtNodeArrayReader);
readingHelper.initWithPtNodeArrayPos(getRootPosition());
bool addedNewUnigram = false;
if (mUpdatingHelper.addUnigramWord(&readingHelper, word, length,
int codePointsToAdd[MAX_WORD_LENGTH];
int codePointCountToAdd = length;
memmove(codePointsToAdd, word, sizeof(int) * length);
if (unigramProperty->representsBeginningOfSentence()) {
codePointCountToAdd = CharUtils::attachBeginningOfSentenceMarker(codePointsToAdd,
codePointCountToAdd, MAX_WORD_LENGTH);
}
if (codePointCountToAdd <= 0) {
return false;
}
if (mUpdatingHelper.addUnigramWord(&readingHelper, codePointsToAdd, codePointCountToAdd,
unigramProperty, &addedNewUnigram)) {
if (addedNewUnigram) {
if (addedNewUnigram && !unigramProperty->representsBeginningOfSentence()) {
mUnigramCount++;
}
if (unigramProperty->getShortcuts().size() > 0) {
@ -221,8 +233,6 @@ bool Ver4PatriciaTriePolicy::addUnigramEntry(const int *const word, const int le
bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsInfo,
const BigramProperty *const bigramProperty) {
const int length0 = prevWordsInfo->getNthPrevWordCodePointCount(1);
const int *word0 = prevWordsInfo->getNthPrevWordCodePoints(1);
if (!mBuffers->isUpdatable()) {
AKLOGI("Warning: addNgramEntry() is called for non-updatable dictionary.");
return false;
@ -232,15 +242,20 @@ bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsI
mDictBuffer->getTailPosition());
return false;
}
if (length0 > MAX_WORD_LENGTH
|| bigramProperty->getTargetCodePoints()->size() > MAX_WORD_LENGTH) {
AKLOGE("Either src word or target word is too long to insert the bigram to the dictionary. "
"length0: %d, length1: %d", length0, bigramProperty->getTargetCodePoints()->size());
if (!prevWordsInfo->isValid()) {
AKLOGE("prev words info is not valid for adding n-gram entry to the dictionary.");
return false;
}
const int word0Pos = getTerminalPtNodePositionOfWord(word0, length0,
false /* forceLowerCaseSearch */);
if (word0Pos == NOT_A_DICT_POS) {
if (bigramProperty->getTargetCodePoints()->size() > MAX_WORD_LENGTH) {
AKLOGE("The word is too long to insert the ngram to the dictionary. "
"length: %d", bigramProperty->getTargetCodePoints()->size());
return false;
}
int prevWordsPtNodePos[MAX_PREV_WORD_COUNT_FOR_N_GRAM];
prevWordsInfo->getPrevWordsTerminalPtNodePos(this, prevWordsPtNodePos,
false /* tryLowerCaseSearch */);
// TODO: Support N-gram.
if (prevWordsPtNodePos[0] == NOT_A_DICT_POS) {
return false;
}
const int word1Pos = getTerminalPtNodePositionOfWord(
@ -250,7 +265,8 @@ bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsI
return false;
}
bool addedNewBigram = false;
if (mUpdatingHelper.addBigramWords(word0Pos, word1Pos, bigramProperty, &addedNewBigram)) {
if (mUpdatingHelper.addBigramWords(prevWordsPtNodePos[0], word1Pos, bigramProperty,
&addedNewBigram)) {
if (addedNewBigram) {
mBigramCount++;
}
@ -261,11 +277,9 @@ bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsI
}
bool Ver4PatriciaTriePolicy::removeNgramEntry(const PrevWordsInfo *const prevWordsInfo,
const int *const word1, const int length1) {
const int length0 = prevWordsInfo->getNthPrevWordCodePointCount(1);
const int *word0 = prevWordsInfo->getNthPrevWordCodePoints(1);
const int *const word, const int length) {
if (!mBuffers->isUpdatable()) {
AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary.");
AKLOGI("Warning: removeNgramEntry() is called for non-updatable dictionary.");
return false;
}
if (mDictBuffer->getTailPosition() >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS) {
@ -273,22 +287,26 @@ bool Ver4PatriciaTriePolicy::removeNgramEntry(const PrevWordsInfo *const prevWor
mDictBuffer->getTailPosition());
return false;
}
if (length0 > MAX_WORD_LENGTH || length1 > MAX_WORD_LENGTH) {
AKLOGE("Either src word or target word is too long to remove the bigram to from the "
"dictionary. length0: %d, length1: %d", length0, length1);
if (!prevWordsInfo->isValid()) {
AKLOGE("prev words info is not valid for removing n-gram entry form the dictionary.");
return false;
}
const int word0Pos = getTerminalPtNodePositionOfWord(word0, length0,
if (length > MAX_WORD_LENGTH) {
AKLOGE("word is too long to remove n-gram entry form the dictionary. length: %d", length);
}
int prevWordsPtNodePos[MAX_PREV_WORD_COUNT_FOR_N_GRAM];
prevWordsInfo->getPrevWordsTerminalPtNodePos(this, prevWordsPtNodePos,
false /* tryLowerCaseSerch */);
// TODO: Support N-gram.
if (prevWordsPtNodePos[0] == NOT_A_DICT_POS) {
return false;
}
const int wordPos = getTerminalPtNodePositionOfWord(word, length,
false /* forceLowerCaseSearch */);
if (word0Pos == NOT_A_DICT_POS) {
if (wordPos == NOT_A_DICT_POS) {
return false;
}
const int word1Pos = getTerminalPtNodePositionOfWord(word1, length1,
false /* forceLowerCaseSearch */);
if (word1Pos == NOT_A_DICT_POS) {
return false;
}
if (mUpdatingHelper.removeBigramWords(word0Pos, word1Pos)) {
if (mUpdatingHelper.removeBigramWords(prevWordsPtNodePos[0], wordPos)) {
mBigramCount--;
return true;
} else {
@ -413,7 +431,7 @@ const WordProperty Ver4PatriciaTriePolicy::getWordProperty(const int *const code
const int probability = bigramEntry.hasHistoricalInfo() ?
ForgettingCurveUtils::decodeProbability(
bigramEntry.getHistoricalInfo(), mHeaderPolicy) :
getProbability(word1Probability, bigramEntry.getProbability());
bigramEntry.getProbability();
bigrams.emplace_back(&word1, probability,
historicalInfo->getTimeStamp(), historicalInfo->getLevel(),
historicalInfo->getCount());
@ -436,7 +454,7 @@ const WordProperty Ver4PatriciaTriePolicy::getWordProperty(const int *const code
shortcuts.emplace_back(&target, shortcutProbability);
}
}
const UnigramProperty unigramProperty(false /* representsBeginningOfSentence */,
const UnigramProperty unigramProperty(ptNodeParams.representsBeginningOfSentence(),
ptNodeParams.isNotAWord(), ptNodeParams.isBlacklisted(), ptNodeParams.getProbability(),
historicalInfo->getTimeStamp(), historicalInfo->getLevel(),
historicalInfo->getCount(), &shortcuts);
@ -479,6 +497,6 @@ int Ver4PatriciaTriePolicy::getNextWordAndNextToken(const int token, int *const
return nextToken;
}
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime

36
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_policy.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h
View File

@ -22,8 +22,8 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
*/
#ifndef LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_POLICY_H
#define LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_POLICY_H
#ifndef LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_POLICY_H
#define LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_POLICY_H
#include <vector>
@ -31,29 +31,29 @@
#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/bigram/ver4_bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/shortcut/ver4_shortcut_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_writer.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_writing_helper.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_pt_node_array_reader.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/bigram/ver4_bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/shortcut/ver4_shortcut_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_writer.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_writing_helper.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_pt_node_array_reader.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
} // namespace v401
} // namespace v402
} // namespace backward
class DicNode;
namespace backward {
namespace v401 {
} // namespace v401
namespace v402 {
} // namespace v402
} // namespace backward
class DicNodeVector;
namespace backward {
namespace v401 {
namespace v402 {
class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
public:
@ -114,8 +114,8 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
bool addNgramEntry(const PrevWordsInfo *const prevWordsInfo,
const BigramProperty *const bigramProperty);
bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, const int *const word,
const int length);
bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, const int *const word1,
const int length1);
bool flush(const char *const filePath);
@ -162,7 +162,7 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
std::vector<int> mTerminalPtNodePositionsForIteratingWords;
mutable bool mIsCorrupted;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif // LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_POLICY_H
#endif // LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_POLICY_H

6
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_reading_utils.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_reading_utils.cpp
View File

@ -21,19 +21,19 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.cpp
*/
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
/* static */ int Ver4PatriciaTrieReadingUtils::getTerminalIdAndAdvancePosition(
const uint8_t *const buffer, int *pos) {
return ByteArrayUtils::readUint32AndAdvancePosition(buffer, pos);
}
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime

14
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_reading_utils.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_reading_utils.h
View File

@ -21,8 +21,8 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.h
*/
#ifndef LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_READING_UTILS_H
#define LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_READING_UTILS_H
#ifndef LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_READING_UTILS_H
#define LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_READING_UTILS_H
#include <cstdint>
@ -30,13 +30,13 @@
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
} // namespace v401
} // namespace v402
} // namespace backward
class BufferWithExtendableBuffer;
namespace backward {
namespace v401 {
namespace v402 {
class Ver4PatriciaTrieReadingUtils {
public:
@ -46,7 +46,7 @@ class Ver4PatriciaTrieReadingUtils {
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4PatriciaTrieReadingUtils);
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_READING_UTILS_H */
#endif /* LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_READING_UTILS_H */

25
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_writing_helper.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_writing_helper.cpp
View File

@ -21,26 +21,26 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp
*/
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_writing_helper.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_writing_helper.h"
#include <cstring>
#include <queue>
#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/bigram/ver4_bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/shortcut/ver4_shortcut_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_writer.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_pt_node_array_reader.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/bigram/ver4_bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/shortcut/ver4_shortcut_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_writer.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_pt_node_array_reader.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/file_utils.h"
#include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
bool Ver4PatriciaTrieWritingHelper::writeToDictFile(const char *const dictDirPath,
const int unigramCount, const int bigramCount) const {
@ -222,13 +222,16 @@ bool Ver4PatriciaTrieWritingHelper::truncateUnigrams(
// Delete unigrams.
while (static_cast<int>(priorityQueue.size()) > maxUnigramCount) {
const int ptNodePos = priorityQueue.top().getDictPos();
priorityQueue.pop();
const PtNodeParams ptNodeParams =
ptNodeReader->fetchNodeInfoInBufferFromPtNodePos(ptNodePos);
if (ptNodeParams.representsNonWordInfo()) {
continue;
}
if (!ptNodeWriter->markPtNodeAsWillBecomeNonTerminal(&ptNodeParams)) {
AKLOGE("Cannot mark PtNode as willBecomeNonterminal. PtNode pos: %d", ptNodePos);
return false;
}
priorityQueue.pop();
}
return true;
}
@ -296,6 +299,6 @@ bool Ver4PatriciaTrieWritingHelper::TraversePolicyToUpdateAllPtNodeFlagsAndTermi
return mPtNodeWriter->updatePtNodeHasBigramsAndShortcutTargetsFlags(ptNodeParams);
}
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime

16
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_writing_helper.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_writing_helper.h
View File

@ -21,22 +21,22 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h
*/
#ifndef LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_WRITING_HELPER_H
#define LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_WRITING_HELPER_H
#ifndef LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_WRITING_HELPER_H
#define LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_WRITING_HELPER_H
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
} // namespace v401
} // namespace v402
} // namespace backward
class HeaderPolicy;
namespace backward {
namespace v401 {
namespace v402 {
class Ver4DictBuffers;
class Ver4PatriciaTrieNodeReader;
class Ver4PatriciaTrieNodeWriter;
@ -133,8 +133,8 @@ class Ver4PatriciaTrieWritingHelper {
Ver4DictBuffers *const mBuffers;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_WRITING_HELPER_H */
#endif /* LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_WRITING_HELPER_H */

6
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_pt_node_array_reader.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_pt_node_array_reader.cpp
View File

@ -21,7 +21,7 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_pt_node_array_reader.cpp
*/
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_pt_node_array_reader.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_pt_node_array_reader.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/patricia_trie_reading_utils.h"
@ -29,7 +29,7 @@
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
bool Ver4PtNodeArrayReader::readPtNodeArrayInfoAndReturnIfValid(const int ptNodeArrayPos,
int *const outPtNodeCount, int *const outFirstPtNodePos) const {
@ -85,6 +85,6 @@ bool Ver4PtNodeArrayReader::readForwardLinkAndReturnIfValid(const int forwordLin
return true;
}
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime

14
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_pt_node_array_reader.h → native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_pt_node_array_reader.h
View File

@ -21,21 +21,21 @@
* suggest/policyimpl/dictionary/structure/v4/ver4_pt_node_array_reader.h
*/
#ifndef LATINIME_BACKWARD_V401_VER4_PT_NODE_ARRAY_READER_H
#define LATINIME_BACKWARD_V401_VER4_PT_NODE_ARRAY_READER_H
#ifndef LATINIME_BACKWARD_V402_VER4_PT_NODE_ARRAY_READER_H
#define LATINIME_BACKWARD_V402_VER4_PT_NODE_ARRAY_READER_H
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_array_reader.h"
namespace latinime {
namespace backward {
namespace v401 {
namespace v402 {
} // namespace v401
} // namespace v402
} // namespace backward
class BufferWithExtendableBuffer;
namespace backward {
namespace v401 {
namespace v402 {
class Ver4PtNodeArrayReader : public PtNodeArrayReader {
public:
@ -51,7 +51,7 @@ class Ver4PtNodeArrayReader : public PtNodeArrayReader {
const BufferWithExtendableBuffer *const mBuffer;
};
} // namespace v401
} // namespace v402
} // namespace backward
} // namespace latinime
#endif /* LATINIME_BACKWARD_V401_VER4_PT_NODE_ARRAY_READER_H */
#endif /* LATINIME_BACKWARD_V402_VER4_PT_NODE_ARRAY_READER_H */

29
native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp
View File

@ -19,9 +19,9 @@
#include <climits>
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h"
#include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h"
@ -57,14 +57,13 @@ namespace latinime {
const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap) {
FormatUtils::FORMAT_VERSION dictFormatVersion = FormatUtils::getFormatVersion(formatVersion);
switch (dictFormatVersion) {
case FormatUtils::VERSION_401: {
return newPolicyForOnMemoryV4Dict<backward::v401::Ver4DictConstants,
backward::v401::Ver4DictBuffers,
backward::v401::Ver4DictBuffers::Ver4DictBuffersPtr,
backward::v401::Ver4PatriciaTriePolicy>(
case FormatUtils::VERSION_4: {
return newPolicyForOnMemoryV4Dict<backward::v402::Ver4DictConstants,
backward::v402::Ver4DictBuffers,
backward::v402::Ver4DictBuffers::Ver4DictBuffersPtr,
backward::v402::Ver4PatriciaTriePolicy>(
dictFormatVersion, locale, attributeMap);
}
case FormatUtils::VERSION_4:
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
case FormatUtils::VERSION_4_DEV: {
return newPolicyForOnMemoryV4Dict<Ver4DictConstants, Ver4DictBuffers,
@ -116,14 +115,13 @@ template<class DictConstants, class DictBuffers, class DictBuffersPtr, class Str
case FormatUtils::VERSION_2:
AKLOGE("Given path is a directory but the format is version 2. path: %s", path);
break;
case FormatUtils::VERSION_401: {
return newPolicyForV4Dict<backward::v401::Ver4DictConstants,
backward::v401::Ver4DictBuffers,
backward::v401::Ver4DictBuffers::Ver4DictBuffersPtr,
backward::v401::Ver4PatriciaTriePolicy>(
case FormatUtils::VERSION_4: {
return newPolicyForV4Dict<backward::v402::Ver4DictConstants,
backward::v402::Ver4DictBuffers,
backward::v402::Ver4DictBuffers::Ver4DictBuffersPtr,
backward::v402::Ver4PatriciaTriePolicy>(
headerFilePath, formatVersion, std::move(mmappedBuffer));
}
case FormatUtils::VERSION_4:
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
case FormatUtils::VERSION_4_DEV: {
return newPolicyForV4Dict<Ver4DictConstants, Ver4DictBuffers,
@ -179,7 +177,6 @@ template<class DictConstants, class DictBuffers, class DictBuffersPtr, class Str
case FormatUtils::VERSION_2:
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(
new PatriciaTriePolicy(std::move(mmappedBuffer)));
case FormatUtils::VERSION_401:
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
case FormatUtils::VERSION_4:
case FormatUtils::VERSION_4_DEV:

11
native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
View File

@ -23,7 +23,7 @@
#include <sys/types.h>
#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
@ -41,12 +41,11 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE =
TimeKeeper::setCurrentTime();
const FormatUtils::FORMAT_VERSION formatVersion = FormatUtils::getFormatVersion(dictVersion);
switch (formatVersion) {
case FormatUtils::VERSION_401:
return createEmptyV4DictFile<backward::v401::Ver4DictConstants,
backward::v401::Ver4DictBuffers,
backward::v401::Ver4DictBuffers::Ver4DictBuffersPtr>(
filePath, localeAsCodePointVector, attributeMap, formatVersion);
case FormatUtils::VERSION_4:
return createEmptyV4DictFile<backward::v402::Ver4DictConstants,
backward::v402::Ver4DictBuffers,
backward::v402::Ver4DictBuffers::Ver4DictBuffersPtr>(
filePath, localeAsCodePointVector, attributeMap, formatVersion);
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
case FormatUtils::VERSION_4_DEV:
return createEmptyV4DictFile<Ver4DictConstants, Ver4DictBuffers,

18
native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp
View File

@ -29,8 +29,6 @@ const int FormatUtils::DICTIONARY_MINIMUM_SIZE = 12;
switch (formatVersion) {
case VERSION_2:
return VERSION_2;
case VERSION_401:
return VERSION_401;
case VERSION_4_ONLY_FOR_TESTING:
return VERSION_4_ONLY_FOR_TESTING;
case VERSION_4:
@ -52,7 +50,7 @@ const int FormatUtils::DICTIONARY_MINIMUM_SIZE = 12;
const uint32_t magicNumber = ByteArrayUtils::readUint32(dict, 0);
switch (magicNumber) {
case MAGIC_NUMBER:
// Version 2 header is as follows:
// The layout of the header is as follows:
// Magic number (4 bytes) 0x9B 0xC1 0x3A 0xFE
// Dictionary format version number (2 bytes)
// Options (2 bytes)
@ -60,19 +58,7 @@ const int FormatUtils::DICTIONARY_MINIMUM_SIZE = 12;
// Conceptually this converts the hardcoded value of the bytes in the file into
// the symbolic value we use in the code. But we want the constants to be the
// same so we use them for both here.
if (ByteArrayUtils::readUint16(dict, 4) == VERSION_2) {
return VERSION_2;
} else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_401) {
return VERSION_401;
} else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_4_ONLY_FOR_TESTING) {
return VERSION_4_ONLY_FOR_TESTING;
} else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_4) {
return VERSION_4;
} else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_4_DEV) {
return VERSION_4_DEV;
} else {
return UNKNOWN_VERSION;
}
return getFormatVersion(ByteArrayUtils::readUint16(dict, 4));
default:
return UNKNOWN_VERSION;
}

1
native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h
View File

@ -32,7 +32,6 @@ class FormatUtils {
// These MUST have the same values as the relevant constants in FormatSpec.java.
VERSION_2 = 2,
VERSION_4_ONLY_FOR_TESTING = 399,
VERSION_401 = 401,
VERSION_4 = 402,
VERSION_4_DEV = 403,
UNKNOWN_VERSION = -1