Merge "Add timestamp as an argument to updating methods."

main
Keisuke Kuroyanagi 2013-11-28 12:02:22 +00:00 committed by Android (Google) Code Review
commit ebd5541edf
27 changed files with 127 additions and 96 deletions

View File

@ -297,7 +297,7 @@ static void latinime_BinaryDictionary_addUnigramWord(JNIEnv *env, jclass clazz,
if (shortcutTarget) { if (shortcutTarget) {
env->GetIntArrayRegion(shortcutTarget, 0, shortcutLength, shortcutTargetCodePoints); env->GetIntArrayRegion(shortcutTarget, 0, shortcutLength, shortcutTargetCodePoints);
} }
dictionary->addUnigramWord(codePoints, wordLength, probability); dictionary->addUnigramWord(codePoints, wordLength, probability, timeStamp);
} }
static void latinime_BinaryDictionary_addBigramWords(JNIEnv *env, jclass clazz, jlong dict, static void latinime_BinaryDictionary_addBigramWords(JNIEnv *env, jclass clazz, jlong dict,
@ -313,7 +313,7 @@ static void latinime_BinaryDictionary_addBigramWords(JNIEnv *env, jclass clazz,
int word1CodePoints[word1Length]; int word1CodePoints[word1Length];
env->GetIntArrayRegion(word1, 0, word1Length, word1CodePoints); env->GetIntArrayRegion(word1, 0, word1Length, word1CodePoints);
dictionary->addBigramWords(word0CodePoints, word0Length, word1CodePoints, dictionary->addBigramWords(word0CodePoints, word0Length, word1CodePoints,
word1Length, probability); word1Length, probability, timeStamp);
} }
static void latinime_BinaryDictionary_removeBigramWords(JNIEnv *env, jclass clazz, jlong dict, static void latinime_BinaryDictionary_removeBigramWords(JNIEnv *env, jclass clazz, jlong dict,
@ -348,13 +348,15 @@ static int latinime_BinaryDictionary_addMultipleDictionaryEntries(JNIEnv *env, j
jclass languageModelParamClass = env->GetObjectClass(languageModelParam); jclass languageModelParamClass = env->GetObjectClass(languageModelParam);
env->DeleteLocalRef(languageModelParam); env->DeleteLocalRef(languageModelParam);
// TODO: Support shortcut, timestamp and flags. // TODO: Support shortcut and flags.
jfieldID word0FieldId = env->GetFieldID(languageModelParamClass, "mWord0", "[I"); jfieldID word0FieldId = env->GetFieldID(languageModelParamClass, "mWord0", "[I");
jfieldID word1FieldId = env->GetFieldID(languageModelParamClass, "mWord1", "[I"); jfieldID word1FieldId = env->GetFieldID(languageModelParamClass, "mWord1", "[I");
jfieldID unigramProbabilityFieldId = jfieldID unigramProbabilityFieldId =
env->GetFieldID(languageModelParamClass, "mUnigramProbability", "I"); env->GetFieldID(languageModelParamClass, "mUnigramProbability", "I");
jfieldID bigramProbabilityFieldId = jfieldID bigramProbabilityFieldId =
env->GetFieldID(languageModelParamClass, "mBigramProbability", "I"); env->GetFieldID(languageModelParamClass, "mBigramProbability", "I");
jfieldID timeStampFieldId =
env->GetFieldID(languageModelParamClass, "mTimeStamp", "I");
env->DeleteLocalRef(languageModelParamClass); env->DeleteLocalRef(languageModelParamClass);
for (int i = startIndex; i < languageModelParamCount; ++i) { for (int i = startIndex; i < languageModelParamCount; ++i) {
@ -375,11 +377,12 @@ static int latinime_BinaryDictionary_addMultipleDictionaryEntries(JNIEnv *env, j
int word1CodePoints[word1Length]; int word1CodePoints[word1Length];
env->GetIntArrayRegion(word1, 0, word1Length, word1CodePoints); env->GetIntArrayRegion(word1, 0, word1Length, word1CodePoints);
jint unigramProbability = env->GetIntField(languageModelParam, unigramProbabilityFieldId); jint unigramProbability = env->GetIntField(languageModelParam, unigramProbabilityFieldId);
dictionary->addUnigramWord(word1CodePoints, word1Length, unigramProbability); jint timeStamp = env->GetIntField(languageModelParam, timeStampFieldId);
dictionary->addUnigramWord(word1CodePoints, word1Length, unigramProbability, timeStamp);
if (word0) { if (word0) {
jint bigramProbability = env->GetIntField(languageModelParam, bigramProbabilityFieldId); jint bigramProbability = env->GetIntField(languageModelParam, bigramProbabilityFieldId);
dictionary->addBigramWords(word0CodePoints, word0Length, word1CodePoints, word1Length, dictionary->addBigramWords(word0CodePoints, word0Length, word1CodePoints, word1Length,
bigramProbability); bigramProbability, timeStamp);
} }
if (dictionary->needsToRunGC(true /* mindsBlockByGC */)) { if (dictionary->needsToRunGC(true /* mindsBlockByGC */)) {
return i + 1; return i + 1;

View File

@ -93,14 +93,16 @@ int Dictionary::getBigramProbability(const int *word0, int length0, const int *w
return mBigramDictionary.get()->getBigramProbability(word0, length0, word1, length1); return mBigramDictionary.get()->getBigramProbability(word0, length0, word1, length1);
} }
void Dictionary::addUnigramWord(const int *const word, const int length, const int probability) { void Dictionary::addUnigramWord(const int *const word, const int length, const int probability,
mDictionaryStructureWithBufferPolicy.get()->addUnigramWord(word, length, probability); const int timestamp) {
mDictionaryStructureWithBufferPolicy.get()->addUnigramWord(word, length, probability,
timestamp);
} }
void Dictionary::addBigramWords(const int *const word0, const int length0, const int *const word1, void Dictionary::addBigramWords(const int *const word0, const int length0, const int *const word1,
const int length1, const int probability) { const int length1, const int probability, const int timestamp) {
mDictionaryStructureWithBufferPolicy.get()->addBigramWords(word0, length0, word1, length1, mDictionaryStructureWithBufferPolicy.get()->addBigramWords(word0, length0, word1, length1,
probability); probability, timestamp);
} }
void Dictionary::removeBigramWords(const int *const word0, const int length0, void Dictionary::removeBigramWords(const int *const word0, const int length0,

View File

@ -72,10 +72,11 @@ class Dictionary {
int getBigramProbability(const int *word0, int length0, const int *word1, int length1) const; int getBigramProbability(const int *word0, int length0, const int *word1, int length1) const;
void addUnigramWord(const int *const word, const int length, const int probability); void addUnigramWord(const int *const word, const int length, const int probability,
const int timestamp);
void addBigramWords(const int *const word0, const int length0, const int *const word1, void addBigramWords(const int *const word0, const int length0, const int *const word1,
const int length1, const int probability); const int length1, const int probability, const int timestamp);
void removeBigramWords(const int *const word0, const int length0, const int *const word1, void removeBigramWords(const int *const word0, const int length0, const int *const word1,
const int length1); const int length1);

View File

@ -67,11 +67,11 @@ class DictionaryStructureWithBufferPolicy {
// Returns whether the update was success or not. // Returns whether the update was success or not.
virtual bool addUnigramWord(const int *const word, const int length, virtual bool addUnigramWord(const int *const word, const int length,
const int probability) = 0; const int probability, const int timestamp) = 0;
// Returns whether the update was success or not. // Returns whether the update was success or not.
virtual bool addBigramWords(const int *const word0, const int length0, const int *const word1, virtual bool addBigramWords(const int *const word0, const int length0, const int *const word1,
const int length1, const int probability) = 0; const int length1, const int probability, const int timestamp) = 0;
// Returns whether the update was success or not. // Returns whether the update was success or not.
virtual bool removeBigramWords(const int *const word0, const int length0, virtual bool removeBigramWords(const int *const word0, const int length0,

View File

@ -36,7 +36,7 @@ void Ver4BigramListPolicy::getNextBigram(int *const outBigramPos, int *const out
} }
bool Ver4BigramListPolicy::addNewEntry(const int terminalId, const int newTargetTerminalId, bool Ver4BigramListPolicy::addNewEntry(const int terminalId, const int newTargetTerminalId,
const int newProbability, bool *const outAddedNewEntry) { const int newProbability, const int timestamp, bool *const outAddedNewEntry) {
if (outAddedNewEntry) { if (outAddedNewEntry) {
*outAddedNewEntry = false; *outAddedNewEntry = false;
} }

View File

@ -44,7 +44,7 @@ class Ver4BigramListPolicy : public DictionaryBigramsStructurePolicy {
} }
bool addNewEntry(const int terminalId, const int newTargetTerminalId, const int newProbability, bool addNewEntry(const int terminalId, const int newTargetTerminalId, const int newProbability,
bool *const outAddedNewEntry); const int timestamp, bool *const outAddedNewEntry);
bool removeEntry(const int terminalId, const int targetTerminalId); bool removeEntry(const int terminalId, const int targetTerminalId);

View File

@ -27,6 +27,9 @@ const char *const HeaderPolicy::LAST_DECAYED_TIME_KEY = "LAST_DECAYED_TIME";
const char *const HeaderPolicy::UNIGRAM_COUNT_KEY = "UNIGRAM_COUNT"; const char *const HeaderPolicy::UNIGRAM_COUNT_KEY = "UNIGRAM_COUNT";
const char *const HeaderPolicy::BIGRAM_COUNT_KEY = "BIGRAM_COUNT"; const char *const HeaderPolicy::BIGRAM_COUNT_KEY = "BIGRAM_COUNT";
const char *const HeaderPolicy::EXTENDED_REGION_SIZE_KEY = "EXTENDED_REGION_SIZE"; const char *const HeaderPolicy::EXTENDED_REGION_SIZE_KEY = "EXTENDED_REGION_SIZE";
// Historical info is information that is needed to support decaying such as timestamp, level and
// count.
const char *const HeaderPolicy::HAS_HISTORICAL_INFO_KEY = "HAS_HISTORICAL_INFO";
const int HeaderPolicy::DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE = 100; const int HeaderPolicy::DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE = 100;
const float HeaderPolicy::MULTIPLE_WORD_COST_MULTIPLIER_SCALE = 100.0f; const float HeaderPolicy::MULTIPLE_WORD_COST_MULTIPLIER_SCALE = 100.0f;

View File

@ -47,7 +47,9 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
mBigramCount(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap, mBigramCount(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
BIGRAM_COUNT_KEY, 0 /* defaultValue */)), BIGRAM_COUNT_KEY, 0 /* defaultValue */)),
mExtendedRegionSize(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap, mExtendedRegionSize(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
EXTENDED_REGION_SIZE_KEY, 0 /* defaultValue */)) {} EXTENDED_REGION_SIZE_KEY, 0 /* defaultValue */)),
mHasHistoricalInfoOfWords(HeaderReadWriteUtils::readBoolAttributeValue(
&mAttributeMap, HAS_HISTORICAL_INFO_KEY, false /* defaultValue */)) {}
// Constructs header information using an attribute map. // Constructs header information using an attribute map.
HeaderPolicy(const FormatUtils::FORMAT_VERSION dictFormatVersion, HeaderPolicy(const FormatUtils::FORMAT_VERSION dictFormatVersion,
@ -62,7 +64,9 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
LAST_UPDATED_TIME_KEY, time(0) /* defaultValue */)), LAST_UPDATED_TIME_KEY, time(0) /* defaultValue */)),
mLastDecayedTime(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap, mLastDecayedTime(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
LAST_UPDATED_TIME_KEY, time(0) /* defaultValue */)), LAST_UPDATED_TIME_KEY, time(0) /* defaultValue */)),
mUnigramCount(0), mBigramCount(0), mExtendedRegionSize(0) {} mUnigramCount(0), mBigramCount(0), mExtendedRegionSize(0),
mHasHistoricalInfoOfWords(HeaderReadWriteUtils::readBoolAttributeValue(
&mAttributeMap, HAS_HISTORICAL_INFO_KEY, false /* defaultValue */)) {}
~HeaderPolicy() {} ~HeaderPolicy() {}
@ -123,6 +127,10 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
return mExtendedRegionSize; return mExtendedRegionSize;
} }
AK_FORCE_INLINE bool hasHistricalInfoOfWords() const {
return mHasHistoricalInfoOfWords;
}
void readHeaderValueOrQuestionMark(const char *const key, void readHeaderValueOrQuestionMark(const char *const key,
int *outValue, int outValueSize) const; int *outValue, int outValueSize) const;
@ -140,6 +148,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
static const char *const UNIGRAM_COUNT_KEY; static const char *const UNIGRAM_COUNT_KEY;
static const char *const BIGRAM_COUNT_KEY; static const char *const BIGRAM_COUNT_KEY;
static const char *const EXTENDED_REGION_SIZE_KEY; static const char *const EXTENDED_REGION_SIZE_KEY;
static const char *const HAS_HISTORICAL_INFO_KEY;
static const int DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE; static const int DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE;
static const float MULTIPLE_WORD_COST_MULTIPLIER_SCALE; static const float MULTIPLE_WORD_COST_MULTIPLIER_SCALE;
@ -154,6 +163,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
const int mUnigramCount; const int mUnigramCount;
const int mBigramCount; const int mBigramCount;
const int mExtendedRegionSize; const int mExtendedRegionSize;
const bool mHasHistoricalInfoOfWords;
float readMultipleWordCostMultiplier() const; float readMultipleWordCostMultiplier() const;

View File

@ -49,7 +49,7 @@ class PtNodeWriter {
const int movedPos, const int bigramLinkedNodePos) = 0; const int movedPos, const int bigramLinkedNodePos) = 0;
virtual bool updatePtNodeProbability(const PtNodeParams *const toBeUpdatedPtNodeParams, virtual bool updatePtNodeProbability(const PtNodeParams *const toBeUpdatedPtNodeParams,
const int probability) = 0; const int probability, const int timestamp) = 0;
virtual bool updateChildrenPosition(const PtNodeParams *const toBeUpdatedPtNodeParams, virtual bool updateChildrenPosition(const PtNodeParams *const toBeUpdatedPtNodeParams,
const int newChildrenPosition) = 0; const int newChildrenPosition) = 0;
@ -58,10 +58,10 @@ class PtNodeWriter {
int *const ptNodeWritingPos) = 0; int *const ptNodeWritingPos) = 0;
virtual bool writeNewTerminalPtNodeAndAdvancePosition(const PtNodeParams *const ptNodeParams, virtual bool writeNewTerminalPtNodeAndAdvancePosition(const PtNodeParams *const ptNodeParams,
int *const ptNodeWritingPos) = 0; const int timestamp, int *const ptNodeWritingPos) = 0;
virtual bool addNewBigramEntry(const PtNodeParams *const sourcePtNodeParams, virtual bool addNewBigramEntry(const PtNodeParams *const sourcePtNodeParams,
const PtNodeParams *const targetPtNodeParam, const int probability, const PtNodeParams *const targetPtNodeParam, const int probability, const int timestamp,
bool *const outAddedNewBigram) = 0; bool *const outAddedNewBigram) = 0;
virtual bool removeBigramEntry(const PtNodeParams *const sourcePtNodeParams, virtual bool removeBigramEntry(const PtNodeParams *const sourcePtNodeParams,

View File

@ -76,14 +76,15 @@ class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
return &mShortcutListPolicy; return &mShortcutListPolicy;
} }
bool addUnigramWord(const int *const word, const int length, const int probability) { bool addUnigramWord(const int *const word, const int length, const int probability,
const int timestamp) {
// This method should not be called for non-updatable dictionary. // This method should not be called for non-updatable dictionary.
AKLOGI("Warning: addUnigramWord() is called for non-updatable dictionary."); AKLOGI("Warning: addUnigramWord() is called for non-updatable dictionary.");
return false; return false;
} }
bool addBigramWords(const int *const word0, const int length0, const int *const word1, bool addBigramWords(const int *const word0, const int length0, const int *const word1,
const int length1, const int probability) { const int length1, const int probability, const int timestamp) {
// This method should not be called for non-updatable dictionary. // This method should not be called for non-updatable dictionary.
AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary."); AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary.");
return false; return false;

View File

@ -30,12 +30,14 @@ bool DynamicPatriciaTrieGcEventListeners
// PtNode is useless when the PtNode is not a terminal and doesn't have any not useless // PtNode is useless when the PtNode is not a terminal and doesn't have any not useless
// children. // children.
bool isUselessPtNode = !ptNodeParams->isTerminal(); bool isUselessPtNode = !ptNodeParams->isTerminal();
if (ptNodeParams->isTerminal() && mIsDecayingDict) { if (ptNodeParams->isTerminal() && mNeedsToDecayWhenUpdating) {
// TODO: Avoid decaying probability during GC.
const int newProbability = const int newProbability =
ForgettingCurveUtils::getEncodedProbabilityToSave(ptNodeParams->getProbability(), ForgettingCurveUtils::getEncodedProbabilityToSave(ptNodeParams->getProbability(),
mHeaderPolicy); mHeaderPolicy);
// Update probability. // Update probability.
if (!mPtNodeWriter->updatePtNodeProbability(ptNodeParams, newProbability)) { if (!mPtNodeWriter->updatePtNodeProbability(ptNodeParams, newProbability,
0 /* timestamp */)) {
return false; return false;
} }
if (!ForgettingCurveUtils::isValidEncodedProbability(newProbability)) { if (!ForgettingCurveUtils::isValidEncodedProbability(newProbability)) {

View File

@ -43,10 +43,10 @@ class DynamicPatriciaTrieGcEventListeners {
TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted( TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted(
const DictionaryHeaderStructurePolicy *const headerPolicy, const DictionaryHeaderStructurePolicy *const headerPolicy,
PtNodeWriter *const ptNodeWriter, BufferWithExtendableBuffer *const buffer, PtNodeWriter *const ptNodeWriter, BufferWithExtendableBuffer *const buffer,
const bool isDecayingDict) const bool needsToDecayWhenUpdating)
: mHeaderPolicy(headerPolicy), mPtNodeWriter(ptNodeWriter), mBuffer(buffer), : mHeaderPolicy(headerPolicy), mPtNodeWriter(ptNodeWriter), mBuffer(buffer),
mIsDecayingDict(isDecayingDict), mValueStack(), mChildrenValue(0), mNeedsToDecayWhenUpdating(needsToDecayWhenUpdating), mValueStack(),
mValidUnigramCount(0) {} mChildrenValue(0), mValidUnigramCount(0) {}
~TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted() {}; ~TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted() {};
@ -80,7 +80,7 @@ class DynamicPatriciaTrieGcEventListeners {
const DictionaryHeaderStructurePolicy *const mHeaderPolicy; const DictionaryHeaderStructurePolicy *const mHeaderPolicy;
PtNodeWriter *const mPtNodeWriter; PtNodeWriter *const mPtNodeWriter;
BufferWithExtendableBuffer *const mBuffer; BufferWithExtendableBuffer *const mBuffer;
const bool mIsDecayingDict; const bool mNeedsToDecayWhenUpdating;
std::vector<int> mValueStack; std::vector<int> mValueStack;
int mChildrenValue; int mChildrenValue;
int mValidUnigramCount; int mValidUnigramCount;

View File

@ -102,7 +102,8 @@ bool DynamicPatriciaTrieNodeWriter::markPtNodeAsMoved(
} }
bool DynamicPatriciaTrieNodeWriter::updatePtNodeProbability( bool DynamicPatriciaTrieNodeWriter::updatePtNodeProbability(
const PtNodeParams *const toBeUpdatedPtNodeParams, const int newProbability) { const PtNodeParams *const toBeUpdatedPtNodeParams, const int newProbability,
const int timestamp) {
if (!toBeUpdatedPtNodeParams->isTerminal()) { if (!toBeUpdatedPtNodeParams->isTerminal()) {
return false; return false;
} }
@ -127,7 +128,7 @@ bool DynamicPatriciaTrieNodeWriter::writePtNodeAndAdvancePosition(
} }
bool DynamicPatriciaTrieNodeWriter::writeNewTerminalPtNodeAndAdvancePosition( bool DynamicPatriciaTrieNodeWriter::writeNewTerminalPtNodeAndAdvancePosition(
const PtNodeParams *const ptNodeParams, int *const ptNodeWritingPos) { const PtNodeParams *const ptNodeParams, const int timestamp, int *const ptNodeWritingPos) {
int probabilityFieldPos = NOT_A_DICT_POS; int probabilityFieldPos = NOT_A_DICT_POS;
if (!writePtNodeAndGetProbabilityFieldPosAndAdvancePosition(ptNodeParams, &probabilityFieldPos, if (!writePtNodeAndGetProbabilityFieldPosAndAdvancePosition(ptNodeParams, &probabilityFieldPos,
ptNodeWritingPos)) { ptNodeWritingPos)) {
@ -144,7 +145,7 @@ bool DynamicPatriciaTrieNodeWriter::writeNewTerminalPtNodeAndAdvancePosition(
bool DynamicPatriciaTrieNodeWriter::addNewBigramEntry( bool DynamicPatriciaTrieNodeWriter::addNewBigramEntry(
const PtNodeParams *const sourcePtNodeParams, const PtNodeParams *const sourcePtNodeParams,
const PtNodeParams *const targetPtNodeParam, const int probability, const PtNodeParams *const targetPtNodeParam, const int probability, const int timestamp,
bool *const outAddedNewBigram) { bool *const outAddedNewBigram) {
const int newNodePos = mBuffer->getTailPosition(); const int newNodePos = mBuffer->getTailPosition();
int writingPos = newNodePos; int writingPos = newNodePos;

View File

@ -52,7 +52,7 @@ class DynamicPatriciaTrieNodeWriter : public PtNodeWriter {
const int movedPos, const int bigramLinkedNodePos); const int movedPos, const int bigramLinkedNodePos);
virtual bool updatePtNodeProbability(const PtNodeParams *const toBeUpdatedPtNodeParams, virtual bool updatePtNodeProbability(const PtNodeParams *const toBeUpdatedPtNodeParams,
const int newProbability); const int newProbability, const int timestamp);
virtual bool updateChildrenPosition(const PtNodeParams *const toBeUpdatedPtNodeParams, virtual bool updateChildrenPosition(const PtNodeParams *const toBeUpdatedPtNodeParams,
const int newChildrenPosition); const int newChildrenPosition);
@ -60,11 +60,11 @@ class DynamicPatriciaTrieNodeWriter : public PtNodeWriter {
virtual bool writePtNodeAndAdvancePosition(const PtNodeParams *const ptNodeParams, virtual bool writePtNodeAndAdvancePosition(const PtNodeParams *const ptNodeParams,
int *const ptNodeWritingPos); int *const ptNodeWritingPos);
virtual bool writeNewTerminalPtNodeAndAdvancePosition( virtual bool writeNewTerminalPtNodeAndAdvancePosition(const PtNodeParams *const ptNodeParams,
const PtNodeParams *const ptNodeParams, int *const ptNodeWritingPos); const int timestamp, int *const ptNodeWritingPos);
virtual bool addNewBigramEntry(const PtNodeParams *const sourcePtNodeParams, virtual bool addNewBigramEntry(const PtNodeParams *const sourcePtNodeParams,
const PtNodeParams *const targetPtNodeParam, const int probability, const PtNodeParams *const targetPtNodeParam, const int probability, const int timestamp,
bool *const outAddedNewBigram); bool *const outAddedNewBigram);
virtual bool removeBigramEntry(const PtNodeParams *const sourcePtNodeParams, virtual bool removeBigramEntry(const PtNodeParams *const sourcePtNodeParams,

View File

@ -141,7 +141,7 @@ int DynamicPatriciaTriePolicy::getBigramsPositionOfPtNode(const int ptNodePos) c
} }
bool DynamicPatriciaTriePolicy::addUnigramWord(const int *const word, const int length, bool DynamicPatriciaTriePolicy::addUnigramWord(const int *const word, const int length,
const int probability) { const int probability, const int timestamp) {
if (!mMmappedBuffer.get()->isUpdatable()) { if (!mMmappedBuffer.get()->isUpdatable()) {
AKLOGI("Warning: addUnigramWord() is called for non-updatable dictionary."); AKLOGI("Warning: addUnigramWord() is called for non-updatable dictionary.");
return false; return false;
@ -154,7 +154,7 @@ bool DynamicPatriciaTriePolicy::addUnigramWord(const int *const word, const int
DynamicPatriciaTrieReadingHelper readingHelper(&mBufferWithExtendableBuffer, &mNodeReader); DynamicPatriciaTrieReadingHelper readingHelper(&mBufferWithExtendableBuffer, &mNodeReader);
readingHelper.initWithPtNodeArrayPos(getRootPosition()); readingHelper.initWithPtNodeArrayPos(getRootPosition());
bool addedNewUnigram = false; bool addedNewUnigram = false;
if (mUpdatingHelper.addUnigramWord(&readingHelper, word, length, probability, if (mUpdatingHelper.addUnigramWord(&readingHelper, word, length, probability, timestamp,
&addedNewUnigram)) { &addedNewUnigram)) {
if (addedNewUnigram) { if (addedNewUnigram) {
mUnigramCount++; mUnigramCount++;
@ -166,7 +166,8 @@ bool DynamicPatriciaTriePolicy::addUnigramWord(const int *const word, const int
} }
bool DynamicPatriciaTriePolicy::addBigramWords(const int *const word0, const int length0, bool DynamicPatriciaTriePolicy::addBigramWords(const int *const word0, const int length0,
const int *const word1, const int length1, const int probability) { const int *const word1, const int length1, const int probability,
const int timestamp) {
if (!mMmappedBuffer.get()->isUpdatable()) { if (!mMmappedBuffer.get()->isUpdatable()) {
AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary."); AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary.");
return false; return false;
@ -187,7 +188,8 @@ bool DynamicPatriciaTriePolicy::addBigramWords(const int *const word0, const int
return false; return false;
} }
bool addedNewBigram = false; bool addedNewBigram = false;
if (mUpdatingHelper.addBigramWords(word0Pos, word1Pos, probability, &addedNewBigram)) { if (mUpdatingHelper.addBigramWords(word0Pos, word1Pos, probability, timestamp,
&addedNewBigram)) {
if (addedNewBigram) { if (addedNewBigram) {
mBigramCount++; mBigramCount++;
} }

View File

@ -88,10 +88,11 @@ class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
return &mShortcutListPolicy; return &mShortcutListPolicy;
} }
bool addUnigramWord(const int *const word, const int length, const int probability); bool addUnigramWord(const int *const word, const int length, const int probability,
const int timestamp);
bool addBigramWords(const int *const word0, const int length0, const int *const word1, bool addBigramWords(const int *const word0, const int length0, const int *const word1,
const int length1, const int probability); const int length1, const int probability, const int timestamp);
bool removeBigramWords(const int *const word0, const int length0, const int *const word1, bool removeBigramWords(const int *const word0, const int length0, const int *const word1,
const int length1); const int length1);

View File

@ -30,7 +30,7 @@ const int DynamicPatriciaTrieUpdatingHelper::CHILDREN_POSITION_FIELD_SIZE = 3;
bool DynamicPatriciaTrieUpdatingHelper::addUnigramWord( bool DynamicPatriciaTrieUpdatingHelper::addUnigramWord(
DynamicPatriciaTrieReadingHelper *const readingHelper, DynamicPatriciaTrieReadingHelper *const readingHelper,
const int *const wordCodePoints, const int codePointCount, const int probability, const int *const wordCodePoints, const int codePointCount, const int probability,
bool *const outAddedNewUnigram) { const int timestamp, bool *const outAddedNewUnigram) {
int parentPos = NOT_A_DICT_POS; int parentPos = NOT_A_DICT_POS;
while (!readingHelper->isEnd()) { while (!readingHelper->isEnd()) {
const PtNodeParams ptNodeParams(readingHelper->getPtNodeParams()); const PtNodeParams ptNodeParams(readingHelper->getPtNodeParams());
@ -52,18 +52,18 @@ bool DynamicPatriciaTrieUpdatingHelper::addUnigramWord(
if (nextIndex >= codePointCount || !readingHelper->isMatchedCodePoint(ptNodeParams, j, if (nextIndex >= codePointCount || !readingHelper->isMatchedCodePoint(ptNodeParams, j,
wordCodePoints[matchedCodePointCount + j])) { wordCodePoints[matchedCodePointCount + j])) {
*outAddedNewUnigram = true; *outAddedNewUnigram = true;
return reallocatePtNodeAndAddNewPtNodes(&ptNodeParams, j, probability, return reallocatePtNodeAndAddNewPtNodes(&ptNodeParams, j, probability, timestamp,
wordCodePoints + matchedCodePointCount, wordCodePoints + matchedCodePointCount,
codePointCount - matchedCodePointCount); codePointCount - matchedCodePointCount);
} }
} }
// All characters are matched. // All characters are matched.
if (codePointCount == readingHelper->getTotalCodePointCount(ptNodeParams)) { if (codePointCount == readingHelper->getTotalCodePointCount(ptNodeParams)) {
return setPtNodeProbability(&ptNodeParams, probability, outAddedNewUnigram); return setPtNodeProbability(&ptNodeParams, probability, timestamp, outAddedNewUnigram);
} }
if (!ptNodeParams.hasChildren()) { if (!ptNodeParams.hasChildren()) {
*outAddedNewUnigram = true; *outAddedNewUnigram = true;
return createChildrenPtNodeArrayAndAChildPtNode(&ptNodeParams, probability, return createChildrenPtNodeArrayAndAChildPtNode(&ptNodeParams, probability, timestamp,
wordCodePoints + readingHelper->getTotalCodePointCount(ptNodeParams), wordCodePoints + readingHelper->getTotalCodePointCount(ptNodeParams),
codePointCount - readingHelper->getTotalCodePointCount(ptNodeParams)); codePointCount - readingHelper->getTotalCodePointCount(ptNodeParams));
} }
@ -79,17 +79,18 @@ bool DynamicPatriciaTrieUpdatingHelper::addUnigramWord(
*outAddedNewUnigram = true; *outAddedNewUnigram = true;
return createAndInsertNodeIntoPtNodeArray(parentPos, return createAndInsertNodeIntoPtNodeArray(parentPos,
wordCodePoints + readingHelper->getPrevTotalCodePointCount(), wordCodePoints + readingHelper->getPrevTotalCodePointCount(),
codePointCount - readingHelper->getPrevTotalCodePointCount(), probability, &pos); codePointCount - readingHelper->getPrevTotalCodePointCount(), probability,
timestamp, &pos);
} }
bool DynamicPatriciaTrieUpdatingHelper::addBigramWords(const int word0Pos, const int word1Pos, bool DynamicPatriciaTrieUpdatingHelper::addBigramWords(const int word0Pos, const int word1Pos,
const int probability, bool *const outAddedNewBigram) { const int probability, const int timestamp, bool *const outAddedNewBigram) {
const PtNodeParams sourcePtNodeParams( const PtNodeParams sourcePtNodeParams(
mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(word0Pos)); mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(word0Pos));
const PtNodeParams targetPtNodeParams( const PtNodeParams targetPtNodeParams(
mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(word1Pos)); mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(word1Pos));
return mPtNodeWriter->addNewBigramEntry(&sourcePtNodeParams, &targetPtNodeParams, probability, return mPtNodeWriter->addNewBigramEntry(&sourcePtNodeParams, &targetPtNodeParams, probability,
outAddedNewBigram); timestamp, outAddedNewBigram);
} }
// Remove a bigram relation from word0Pos to word1Pos. // Remove a bigram relation from word0Pos to word1Pos.
@ -103,23 +104,23 @@ bool DynamicPatriciaTrieUpdatingHelper::removeBigramWords(const int word0Pos, co
bool DynamicPatriciaTrieUpdatingHelper::createAndInsertNodeIntoPtNodeArray(const int parentPos, bool DynamicPatriciaTrieUpdatingHelper::createAndInsertNodeIntoPtNodeArray(const int parentPos,
const int *const nodeCodePoints, const int nodeCodePointCount, const int probability, const int *const nodeCodePoints, const int nodeCodePointCount, const int probability,
int *const forwardLinkFieldPos) { const int timestamp, int *const forwardLinkFieldPos) {
const int newPtNodeArrayPos = mBuffer->getTailPosition(); const int newPtNodeArrayPos = mBuffer->getTailPosition();
if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer,
newPtNodeArrayPos, forwardLinkFieldPos)) { newPtNodeArrayPos, forwardLinkFieldPos)) {
return false; return false;
} }
return createNewPtNodeArrayWithAChildPtNode(parentPos, nodeCodePoints, nodeCodePointCount, return createNewPtNodeArrayWithAChildPtNode(parentPos, nodeCodePoints, nodeCodePointCount,
probability); probability, timestamp);
} }
bool DynamicPatriciaTrieUpdatingHelper::setPtNodeProbability( bool DynamicPatriciaTrieUpdatingHelper::setPtNodeProbability(
const PtNodeParams *const originalPtNodeParams, const int probability, const PtNodeParams *const originalPtNodeParams, const int probability, const int timestamp,
bool *const outAddedNewUnigram) { bool *const outAddedNewUnigram) {
if (originalPtNodeParams->isTerminal()) { if (originalPtNodeParams->isTerminal()) {
// Overwrites the probability. // Overwrites the probability.
*outAddedNewUnigram = false; *outAddedNewUnigram = false;
return mPtNodeWriter->updatePtNodeProbability(originalPtNodeParams, probability); return mPtNodeWriter->updatePtNodeProbability(originalPtNodeParams, probability, timestamp);
} else { } else {
// Make the node terminal and write the probability. // Make the node terminal and write the probability.
*outAddedNewUnigram = true; *outAddedNewUnigram = true;
@ -130,7 +131,7 @@ bool DynamicPatriciaTrieUpdatingHelper::setPtNodeProbability(
originalPtNodeParams->getCodePointCount(), originalPtNodeParams->getCodePoints(), originalPtNodeParams->getCodePointCount(), originalPtNodeParams->getCodePoints(),
probability)); probability));
if (!mPtNodeWriter->writeNewTerminalPtNodeAndAdvancePosition(&ptNodeParamsToWrite, if (!mPtNodeWriter->writeNewTerminalPtNodeAndAdvancePosition(&ptNodeParamsToWrite,
&writingPos)) { timestamp, &writingPos)) {
return false; return false;
} }
if (!mPtNodeWriter->markPtNodeAsMoved(originalPtNodeParams, movedPos, movedPos)) { if (!mPtNodeWriter->markPtNodeAsMoved(originalPtNodeParams, movedPos, movedPos)) {
@ -141,19 +142,19 @@ bool DynamicPatriciaTrieUpdatingHelper::setPtNodeProbability(
} }
bool DynamicPatriciaTrieUpdatingHelper::createChildrenPtNodeArrayAndAChildPtNode( bool DynamicPatriciaTrieUpdatingHelper::createChildrenPtNodeArrayAndAChildPtNode(
const PtNodeParams *const parentPtNodeParams, const int probability, const PtNodeParams *const parentPtNodeParams, const int probability, const int timestamp,
const int *const codePoints, const int codePointCount) { const int *const codePoints, const int codePointCount) {
const int newPtNodeArrayPos = mBuffer->getTailPosition(); const int newPtNodeArrayPos = mBuffer->getTailPosition();
if (!mPtNodeWriter->updateChildrenPosition(parentPtNodeParams, newPtNodeArrayPos)) { if (!mPtNodeWriter->updateChildrenPosition(parentPtNodeParams, newPtNodeArrayPos)) {
return false; return false;
} }
return createNewPtNodeArrayWithAChildPtNode(parentPtNodeParams->getHeadPos(), codePoints, return createNewPtNodeArrayWithAChildPtNode(parentPtNodeParams->getHeadPos(), codePoints,
codePointCount, probability); codePointCount, probability, timestamp);
} }
bool DynamicPatriciaTrieUpdatingHelper::createNewPtNodeArrayWithAChildPtNode( bool DynamicPatriciaTrieUpdatingHelper::createNewPtNodeArrayWithAChildPtNode(
const int parentPtNodePos, const int *const nodeCodePoints, const int nodeCodePointCount, const int parentPtNodePos, const int *const nodeCodePoints, const int nodeCodePointCount,
const int probability) { const int probability, const int timestamp) {
int writingPos = mBuffer->getTailPosition(); int writingPos = mBuffer->getTailPosition();
if (!DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer, if (!DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer,
1 /* arraySize */, &writingPos)) { 1 /* arraySize */, &writingPos)) {
@ -161,7 +162,7 @@ bool DynamicPatriciaTrieUpdatingHelper::createNewPtNodeArrayWithAChildPtNode(
} }
const PtNodeParams ptNodeParamsToWrite(getPtNodeParamsForNewPtNode(true /* isTerminal */, const PtNodeParams ptNodeParamsToWrite(getPtNodeParamsForNewPtNode(true /* isTerminal */,
parentPtNodePos, nodeCodePointCount, nodeCodePoints, probability)); parentPtNodePos, nodeCodePointCount, nodeCodePoints, probability));
if (!mPtNodeWriter->writeNewTerminalPtNodeAndAdvancePosition(&ptNodeParamsToWrite, if (!mPtNodeWriter->writeNewTerminalPtNodeAndAdvancePosition(&ptNodeParamsToWrite, timestamp,
&writingPos)) { &writingPos)) {
return false; return false;
} }
@ -175,7 +176,7 @@ bool DynamicPatriciaTrieUpdatingHelper::createNewPtNodeArrayWithAChildPtNode(
// Returns whether the dictionary updating was succeeded or not. // Returns whether the dictionary updating was succeeded or not.
bool DynamicPatriciaTrieUpdatingHelper::reallocatePtNodeAndAddNewPtNodes( bool DynamicPatriciaTrieUpdatingHelper::reallocatePtNodeAndAddNewPtNodes(
const PtNodeParams *const reallocatingPtNodeParams, const int overlappingCodePointCount, const PtNodeParams *const reallocatingPtNodeParams, const int overlappingCodePointCount,
const int probabilityOfNewPtNode, const int *const newNodeCodePoints, const int probabilityOfNewPtNode, const int timestamp, const int *const newNodeCodePoints,
const int newNodeCodePointCount) { const int newNodeCodePointCount) {
// When addsExtraChild is true, split the reallocating PtNode and add new child. // When addsExtraChild is true, split the reallocating PtNode and add new child.
// Reallocating PtNode: abcde, newNode: abcxy. // Reallocating PtNode: abcde, newNode: abcxy.
@ -201,7 +202,7 @@ bool DynamicPatriciaTrieUpdatingHelper::reallocatePtNodeAndAddNewPtNodes(
reallocatingPtNodeParams->getParentPos(), overlappingCodePointCount, reallocatingPtNodeParams->getParentPos(), overlappingCodePointCount,
reallocatingPtNodeParams->getCodePoints(), probabilityOfNewPtNode)); reallocatingPtNodeParams->getCodePoints(), probabilityOfNewPtNode));
if (!mPtNodeWriter->writeNewTerminalPtNodeAndAdvancePosition(&ptNodeParamsToWrite, if (!mPtNodeWriter->writeNewTerminalPtNodeAndAdvancePosition(&ptNodeParamsToWrite,
&writingPos)) { timestamp, &writingPos)) {
return false; return false;
} }
} }
@ -227,7 +228,7 @@ bool DynamicPatriciaTrieUpdatingHelper::reallocatePtNodeAndAddNewPtNodes(
firstPartOfReallocatedPtNodePos, newNodeCodePointCount - overlappingCodePointCount, firstPartOfReallocatedPtNodePos, newNodeCodePointCount - overlappingCodePointCount,
newNodeCodePoints + overlappingCodePointCount, probabilityOfNewPtNode)); newNodeCodePoints + overlappingCodePointCount, probabilityOfNewPtNode));
if (!mPtNodeWriter->writeNewTerminalPtNodeAndAdvancePosition(&extraChildPtNodeParams, if (!mPtNodeWriter->writeNewTerminalPtNodeAndAdvancePosition(&extraChildPtNodeParams,
&writingPos)) { timestamp, &writingPos)) {
return false; return false;
} }
} }

View File

@ -42,11 +42,11 @@ class DynamicPatriciaTrieUpdatingHelper {
// Add a word to the dictionary. If the word already exists, update the probability. // Add a word to the dictionary. If the word already exists, update the probability.
bool addUnigramWord(DynamicPatriciaTrieReadingHelper *const readingHelper, bool addUnigramWord(DynamicPatriciaTrieReadingHelper *const readingHelper,
const int *const wordCodePoints, const int codePointCount, const int probability, const int *const wordCodePoints, const int codePointCount, const int probability,
bool *const outAddedNewUnigram); const int timestamp, bool *const outAddedNewUnigram);
// Add a bigram relation from word0Pos to word1Pos. // Add a bigram relation from word0Pos to word1Pos.
bool addBigramWords(const int word0Pos, const int word1Pos, const int probability, bool addBigramWords(const int word0Pos, const int word1Pos, const int probability,
bool *const outAddedNewBigram); const int timestamp, bool *const outAddedNewBigram);
// Remove a bigram relation from word0Pos to word1Pos. // Remove a bigram relation from word0Pos to word1Pos.
bool removeBigramWords(const int word0Pos, const int word1Pos); bool removeBigramWords(const int word0Pos, const int word1Pos);
@ -61,21 +61,23 @@ class DynamicPatriciaTrieUpdatingHelper {
PtNodeWriter *const mPtNodeWriter; PtNodeWriter *const mPtNodeWriter;
bool createAndInsertNodeIntoPtNodeArray(const int parentPos, const int *const nodeCodePoints, bool createAndInsertNodeIntoPtNodeArray(const int parentPos, const int *const nodeCodePoints,
const int nodeCodePointCount, const int probability, int *const forwardLinkFieldPos); const int nodeCodePointCount, const int probability, const int timestamp,
int *const forwardLinkFieldPos);
bool setPtNodeProbability(const PtNodeParams *const originalPtNodeParams, const int probability, bool setPtNodeProbability(const PtNodeParams *const originalPtNodeParams, const int probability,
bool *const outAddedNewUnigram); const int timestamp, bool *const outAddedNewUnigram);
bool createChildrenPtNodeArrayAndAChildPtNode(const PtNodeParams *const parentPtNodeParams, bool createChildrenPtNodeArrayAndAChildPtNode(const PtNodeParams *const parentPtNodeParams,
const int probability, const int *const codePoints, const int codePointCount); const int probability, const int timestamp, const int *const codePoints,
const int codePointCount);
bool createNewPtNodeArrayWithAChildPtNode(const int parentPos, const int *const nodeCodePoints, bool createNewPtNodeArrayWithAChildPtNode(const int parentPos, const int *const nodeCodePoints,
const int nodeCodePointCount, const int probability); const int nodeCodePointCount, const int probability, const int timestamp);
bool reallocatePtNodeAndAddNewPtNodes( bool reallocatePtNodeAndAddNewPtNodes(
const PtNodeParams *const reallocatingPtNodeParams, const int overlappingCodePointCount, const PtNodeParams *const reallocatingPtNodeParams, const int overlappingCodePointCount,
const int probabilityOfNewPtNode, const int *const newNodeCodePoints, const int probabilityOfNewPtNode, const int timestamp,
const int newNodeCodePointCount); const int *const newNodeCodePoints, const int newNodeCodePointCount);
const PtNodeParams getUpdatedPtNodeParams(const PtNodeParams *const originalPtNodeParams, const PtNodeParams getUpdatedPtNodeParams(const PtNodeParams *const originalPtNodeParams,
const bool isTerminal, const int parentPos, const int codePointCount, const bool isTerminal, const int parentPos, const int codePointCount,

View File

@ -28,7 +28,9 @@ int ProbabilityDictContent::getProbability(const int terminalId) const {
if (terminalId < 0 || terminalId >= mSize) { if (terminalId < 0 || terminalId >= mSize) {
return NOT_A_PROBABILITY; return NOT_A_PROBABILITY;
} }
return Ver4PatriciaTrieReadingUtils::getProbability(getBuffer(), terminalId); const int probabilityFieldPos =
getEntryPos(terminalId) + Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE;
return getBuffer()->readUint(Ver4DictConstants::PROBABILITY_SIZE, probabilityFieldPos);
} }
bool ProbabilityDictContent::setProbability(const int terminalId, const int probability) { bool ProbabilityDictContent::setProbability(const int terminalId, const int probability) {

View File

@ -38,11 +38,13 @@ const int Ver4DictConstants::MAX_DICTIONARY_SIZE = 2 * 1024 * 1024;
const int Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE = 1 * 1024 * 1024; const int Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE = 1 * 1024 * 1024;
const int Ver4DictConstants::NOT_A_TERMINAL_ID = -1; const int Ver4DictConstants::NOT_A_TERMINAL_ID = -1;
const int Ver4DictConstants::NOT_A_TIME_STAMP = -1;
const int Ver4DictConstants::PROBABILITY_SIZE = 1; const int Ver4DictConstants::PROBABILITY_SIZE = 1;
const int Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE = 1; const int Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE = 1;
const int Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE = 3; const int Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE = 3;
const int Ver4DictConstants::NOT_A_TERMINAL_ADDRESS = 0; const int Ver4DictConstants::NOT_A_TERMINAL_ADDRESS = 0;
const int Ver4DictConstants::TERMINAL_ID_FIELD_SIZE = 4; const int Ver4DictConstants::TERMINAL_ID_FIELD_SIZE = 4;
const int Ver4DictConstants::TIME_STAMP_FIELD_SIZE = 4;
const int Ver4DictConstants::BIGRAM_ADDRESS_TABLE_BLOCK_SIZE = 4; const int Ver4DictConstants::BIGRAM_ADDRESS_TABLE_BLOCK_SIZE = 4;
const int Ver4DictConstants::BIGRAM_ADDRESS_TABLE_DATA_SIZE = 4; const int Ver4DictConstants::BIGRAM_ADDRESS_TABLE_DATA_SIZE = 4;
@ -57,6 +59,7 @@ const int Ver4DictConstants::INVALID_BIGRAM_TARGET_TERMINAL_ID =
const int Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE = 1; const int Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE = 1;
const int Ver4DictConstants::BIGRAM_PROBABILITY_MASK = 0x0F; const int Ver4DictConstants::BIGRAM_PROBABILITY_MASK = 0x0F;
const int Ver4DictConstants::BIGRAM_HAS_NEXT_MASK = 0x80; const int Ver4DictConstants::BIGRAM_HAS_NEXT_MASK = 0x80;
const int Ver4DictConstants::BIGRAM_LARGE_PROBABILITY_FIELD_SIZE = 1;
const int Ver4DictConstants::SHORTCUT_FLAGS_FIELD_SIZE = 1; const int Ver4DictConstants::SHORTCUT_FLAGS_FIELD_SIZE = 1;

View File

@ -38,11 +38,15 @@ class Ver4DictConstants {
static const int MAX_DICT_EXTENDED_REGION_SIZE; static const int MAX_DICT_EXTENDED_REGION_SIZE;
static const int NOT_A_TERMINAL_ID; static const int NOT_A_TERMINAL_ID;
static const int NOT_A_TIME_STAMP;
static const int PROBABILITY_SIZE; static const int PROBABILITY_SIZE;
static const int FLAGS_IN_PROBABILITY_FILE_SIZE; static const int FLAGS_IN_PROBABILITY_FILE_SIZE;
static const int TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE; static const int TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE;
static const int NOT_A_TERMINAL_ADDRESS; static const int NOT_A_TERMINAL_ADDRESS;
static const int TERMINAL_ID_FIELD_SIZE; static const int TERMINAL_ID_FIELD_SIZE;
static const int TIME_STAMP_FIELD_SIZE;
static const int WORD_LEVEL_FIELD_SIZE;
static const int WORD_COUNT_FIELD_SIZE;
static const int BIGRAM_ADDRESS_TABLE_BLOCK_SIZE; static const int BIGRAM_ADDRESS_TABLE_BLOCK_SIZE;
static const int BIGRAM_ADDRESS_TABLE_DATA_SIZE; static const int BIGRAM_ADDRESS_TABLE_DATA_SIZE;
@ -54,6 +58,8 @@ class Ver4DictConstants {
static const int INVALID_BIGRAM_TARGET_TERMINAL_ID; static const int INVALID_BIGRAM_TARGET_TERMINAL_ID;
static const int BIGRAM_PROBABILITY_MASK; static const int BIGRAM_PROBABILITY_MASK;
static const int BIGRAM_HAS_NEXT_MASK; static const int BIGRAM_HAS_NEXT_MASK;
// Used when bigram list has time stamp.
static const int BIGRAM_LARGE_PROBABILITY_FIELD_SIZE;
static const int SHORTCUT_FLAGS_FIELD_SIZE; static const int SHORTCUT_FLAGS_FIELD_SIZE;

View File

@ -112,7 +112,8 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsMoved(
} }
bool Ver4PatriciaTrieNodeWriter::updatePtNodeProbability( bool Ver4PatriciaTrieNodeWriter::updatePtNodeProbability(
const PtNodeParams *const toBeUpdatedPtNodeParams, const int newProbability) { const PtNodeParams *const toBeUpdatedPtNodeParams, const int newProbability,
const int timestamp) {
if (!toBeUpdatedPtNodeParams->isTerminal()) { if (!toBeUpdatedPtNodeParams->isTerminal()) {
return false; return false;
} }
@ -143,7 +144,7 @@ bool Ver4PatriciaTrieNodeWriter::writePtNodeAndAdvancePosition(
bool Ver4PatriciaTrieNodeWriter::writeNewTerminalPtNodeAndAdvancePosition( bool Ver4PatriciaTrieNodeWriter::writeNewTerminalPtNodeAndAdvancePosition(
const PtNodeParams *const ptNodeParams, int *const ptNodeWritingPos) { const PtNodeParams *const ptNodeParams, const int timestamp, int *const ptNodeWritingPos) {
int terminalId = Ver4DictConstants::NOT_A_TERMINAL_ID; int terminalId = Ver4DictConstants::NOT_A_TERMINAL_ID;
if (!writePtNodeAndGetTerminalIdAndAdvancePosition(ptNodeParams, &terminalId, if (!writePtNodeAndGetTerminalIdAndAdvancePosition(ptNodeParams, &terminalId,
ptNodeWritingPos)) { ptNodeWritingPos)) {
@ -158,10 +159,10 @@ bool Ver4PatriciaTrieNodeWriter::writeNewTerminalPtNodeAndAdvancePosition(
bool Ver4PatriciaTrieNodeWriter::addNewBigramEntry( bool Ver4PatriciaTrieNodeWriter::addNewBigramEntry(
const PtNodeParams *const sourcePtNodeParams, const PtNodeParams *const sourcePtNodeParams,
const PtNodeParams *const targetPtNodeParam, const int probability, const PtNodeParams *const targetPtNodeParam, const int probability, const int timestamp,
bool *const outAddedNewBigram) { bool *const outAddedNewBigram) {
return mBigramPolicy->addNewEntry(sourcePtNodeParams->getTerminalId(), return mBigramPolicy->addNewEntry(sourcePtNodeParams->getTerminalId(),
targetPtNodeParam->getTerminalId(), probability, outAddedNewBigram); targetPtNodeParam->getTerminalId(), probability, timestamp, outAddedNewBigram);
} }
bool Ver4PatriciaTrieNodeWriter::removeBigramEntry( bool Ver4PatriciaTrieNodeWriter::removeBigramEntry(

View File

@ -54,7 +54,7 @@ class Ver4PatriciaTrieNodeWriter : public PtNodeWriter {
const int movedPos, const int bigramLinkedNodePos); const int movedPos, const int bigramLinkedNodePos);
virtual bool updatePtNodeProbability(const PtNodeParams *const toBeUpdatedPtNodeParams, virtual bool updatePtNodeProbability(const PtNodeParams *const toBeUpdatedPtNodeParams,
const int newProbability); const int newProbability, const int timestamp);
virtual bool updateChildrenPosition(const PtNodeParams *const toBeUpdatedPtNodeParams, virtual bool updateChildrenPosition(const PtNodeParams *const toBeUpdatedPtNodeParams,
const int newChildrenPosition); const int newChildrenPosition);
@ -65,11 +65,11 @@ class Ver4PatriciaTrieNodeWriter : public PtNodeWriter {
virtual bool writePtNodeAndAdvancePosition(const PtNodeParams *const ptNodeParams, virtual bool writePtNodeAndAdvancePosition(const PtNodeParams *const ptNodeParams,
int *const ptNodeWritingPos); int *const ptNodeWritingPos);
virtual bool writeNewTerminalPtNodeAndAdvancePosition( virtual bool writeNewTerminalPtNodeAndAdvancePosition(const PtNodeParams *const ptNodeParams,
const PtNodeParams *const ptNodeParams, int *const ptNodeWritingPos); const int timestamp, int *const ptNodeWritingPos);
virtual bool addNewBigramEntry(const PtNodeParams *const sourcePtNodeParams, virtual bool addNewBigramEntry(const PtNodeParams *const sourcePtNodeParams,
const PtNodeParams *const targetPtNodeParam, const int probability, const PtNodeParams *const targetPtNodeParam, const int probability, const int timestamp,
bool *const outAddedNewBigram); bool *const outAddedNewBigram);
virtual bool removeBigramEntry(const PtNodeParams *const sourcePtNodeParams, virtual bool removeBigramEntry(const PtNodeParams *const sourcePtNodeParams,

View File

@ -137,7 +137,7 @@ int Ver4PatriciaTriePolicy::getBigramsPositionOfPtNode(const int ptNodePos) cons
} }
bool Ver4PatriciaTriePolicy::addUnigramWord(const int *const word, const int length, bool Ver4PatriciaTriePolicy::addUnigramWord(const int *const word, const int length,
const int probability) { const int probability, const int timestamp) {
if (!mBuffers.get()->isUpdatable()) { if (!mBuffers.get()->isUpdatable()) {
AKLOGI("Warning: addUnigramWord() is called for non-updatable dictionary."); AKLOGI("Warning: addUnigramWord() is called for non-updatable dictionary.");
return false; return false;
@ -150,7 +150,7 @@ bool Ver4PatriciaTriePolicy::addUnigramWord(const int *const word, const int len
DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader); DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader);
readingHelper.initWithPtNodeArrayPos(getRootPosition()); readingHelper.initWithPtNodeArrayPos(getRootPosition());
bool addedNewUnigram = false; bool addedNewUnigram = false;
if (mUpdatingHelper.addUnigramWord(&readingHelper, word, length, probability, if (mUpdatingHelper.addUnigramWord(&readingHelper, word, length, probability, timestamp,
&addedNewUnigram)) { &addedNewUnigram)) {
if (addedNewUnigram) { if (addedNewUnigram) {
mUnigramCount++; mUnigramCount++;
@ -162,7 +162,8 @@ bool Ver4PatriciaTriePolicy::addUnigramWord(const int *const word, const int len
} }
bool Ver4PatriciaTriePolicy::addBigramWords(const int *const word0, const int length0, bool Ver4PatriciaTriePolicy::addBigramWords(const int *const word0, const int length0,
const int *const word1, const int length1, const int probability) { const int *const word1, const int length1, const int probability,
const int timestamp) {
if (!mBuffers.get()->isUpdatable()) { if (!mBuffers.get()->isUpdatable()) {
AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary."); AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary.");
return false; return false;
@ -183,7 +184,8 @@ bool Ver4PatriciaTriePolicy::addBigramWords(const int *const word0, const int le
return false; return false;
} }
bool addedNewBigram = false; bool addedNewBigram = false;
if (mUpdatingHelper.addBigramWords(word0Pos, word1Pos, probability, &addedNewBigram)) { if (mUpdatingHelper.addBigramWords(word0Pos, word1Pos, probability, timestamp,
&addedNewBigram)) {
if (addedNewBigram) { if (addedNewBigram) {
mBigramCount++; mBigramCount++;
} }

View File

@ -89,10 +89,11 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
return &mShortcutPolicy; return &mShortcutPolicy;
} }
bool addUnigramWord(const int *const word, const int length, const int probability); bool addUnigramWord(const int *const word, const int length, const int probability,
const int timestamp);
bool addBigramWords(const int *const word0, const int length0, const int *const word1, bool addBigramWords(const int *const word0, const int length0, const int *const word1,
const int length1, const int probability); const int length1, const int probability, const int timestamp);
bool removeBigramWords(const int *const word0, const int length0, const int *const word1, bool removeBigramWords(const int *const word0, const int length0, const int *const word1,
const int length1); const int length1);

View File

@ -16,8 +16,6 @@
#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" #include "suggest/policyimpl/dictionary/utils/byte_array_utils.h"
namespace latinime { namespace latinime {
@ -27,12 +25,4 @@ namespace latinime {
return ByteArrayUtils::readUint32AndAdvancePosition(buffer, pos); return ByteArrayUtils::readUint32AndAdvancePosition(buffer, pos);
} }
/* static */ int Ver4PatriciaTrieReadingUtils::getProbability(
const BufferWithExtendableBuffer *const probabilityBuffer, const int terminalId) {
const int pos = terminalId * (Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE
+ Ver4DictConstants::PROBABILITY_SIZE)
+ Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE;
return probabilityBuffer->readUint(Ver4DictConstants::PROBABILITY_SIZE, pos);
}
} // namespace latinime } // namespace latinime

View File

@ -30,9 +30,6 @@ class Ver4PatriciaTrieReadingUtils {
static int getTerminalIdAndAdvancePosition(const uint8_t *const buffer, static int getTerminalIdAndAdvancePosition(const uint8_t *const buffer,
int *const pos); int *const pos);
static int getProbability(const BufferWithExtendableBuffer *const probabilityBuffer,
const int terminalId);
private: private:
DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4PatriciaTrieReadingUtils); DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4PatriciaTrieReadingUtils);
}; };