diff --git a/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h b/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h index 59748c80d..a8dab9fcd 100644 --- a/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h +++ b/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h @@ -44,8 +44,6 @@ class DictionaryHeaderStructurePolicy { virtual float getMultiWordCostMultiplier() const = 0; - virtual int getLastDecayedTime() const = 0; - virtual void readHeaderValueOrQuestionMark(const char *const key, int *outValue, int outValueSize) const = 0; diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.cpp index cd2243025..7f916677a 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.cpp @@ -37,7 +37,8 @@ void Ver4BigramListPolicy::getNextBigram(int *const outBigramPos, int *const out if (outProbability) { if (bigramEntry.hasHistoricalInfo()) { *outProbability = - ForgettingCurveUtils::decodeProbability(bigramEntry.getHistoricalInfo()); + ForgettingCurveUtils::decodeProbability(bigramEntry.getHistoricalInfo(), + mHeaderPolicy); } else { *outProbability = bigramEntry.getProbability(); } @@ -160,7 +161,7 @@ bool Ver4BigramListPolicy::updateAllBigramEntriesAndDeleteUselessEntries(const i } } else if (bigramEntry.hasHistoricalInfo()) { const HistoricalInfo historicalInfo = ForgettingCurveUtils::createHistoricalInfoToSave( - bigramEntry.getHistoricalInfo()); + bigramEntry.getHistoricalInfo(), mHeaderPolicy); if (ForgettingCurveUtils::needsToKeep(&historicalInfo)) { const BigramEntry updatedBigramEntry = bigramEntry.updateHistoricalInfoAndGetEntry(&historicalInfo); @@ -230,7 +231,8 @@ const BigramEntry Ver4BigramListPolicy::createUpdatedBigramEntryFrom( if (mHeaderPolicy->hasHistoricalInfoOfWords()) { const HistoricalInfo updatedHistoricalInfo = ForgettingCurveUtils::createUpdatedHistoricalInfo( - originalBigramEntry->getHistoricalInfo(), newProbability, timestamp); + originalBigramEntry->getHistoricalInfo(), newProbability, timestamp, + mHeaderPolicy); return originalBigramEntry->updateHistoricalInfoAndGetEntry(&updatedHistoricalInfo); } else { return originalBigramEntry->updateProbabilityAndGetEntry(newProbability); diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp index 3ce57d910..b8a6bbade 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp @@ -35,6 +35,8 @@ const char *const HeaderPolicy::HAS_HISTORICAL_INFO_KEY = "HAS_HISTORICAL_INFO"; const char *const HeaderPolicy::LOCALE_KEY = "locale"; // match Java declaration const int HeaderPolicy::DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE = 100; const float HeaderPolicy::MULTIPLE_WORD_COST_MULTIPLIER_SCALE = 100.0f; +const int HeaderPolicy::DEFAULT_FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP = 4; +const int HeaderPolicy::DEFAULT_FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID = 0; // Used for logging. Question mark is used to indicate that the key is not found. void HeaderPolicy::readHeaderValueOrQuestionMark(const char *const key, int *outValue, diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h index fc347618c..cd5947195 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h @@ -159,6 +159,14 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { return &mAttributeMap; } + AK_FORCE_INLINE int getForgettingCurveOccurrencesToLevelUp() const { + return DEFAULT_FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP; + } + + AK_FORCE_INLINE int getForgettingCurveProbabilityValuesTableId() const { + return DEFAULT_FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID; + } + void readHeaderValueOrQuestionMark(const char *const key, int *outValue, int outValueSize) const; @@ -185,6 +193,8 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { static const char *const LOCALE_KEY; static const int DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE; static const float MULTIPLE_WORD_COST_MULTIPLIER_SCALE; + static const int DEFAULT_FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP; + static const int DEFAULT_FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID; const FormatUtils::FORMAT_VERSION mDictFormatVersion; const HeaderReadWriteUtils::DictionaryFlags mDictionaryFlags; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp index 17fc9483b..f149781f4 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp @@ -65,7 +65,7 @@ const PtNodeParams Ver4PatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProce mProbabilityDictContent->getProbabilityEntry(terminalId); if (probabilityEntry.hasHistoricalInfo()) { probability = ForgettingCurveUtils::decodeProbability( - probabilityEntry.getHistoricalInfo()); + probabilityEntry.getHistoricalInfo(), mHeaderPolicy); } else { probability = probabilityEntry.getProbability(); } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h index 9d932457c..1db9ea026 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h @@ -26,6 +26,7 @@ namespace latinime { class BufferWithExtendableBuffer; +class HeaderPolicy; class ProbabilityDictContent; /* @@ -35,8 +36,10 @@ class ProbabilityDictContent; class Ver4PatriciaTrieNodeReader : public PtNodeReader { public: Ver4PatriciaTrieNodeReader(const BufferWithExtendableBuffer *const buffer, - const ProbabilityDictContent *const probabilityDictContent) - : mBuffer(buffer), mProbabilityDictContent(probabilityDictContent) {} + const ProbabilityDictContent *const probabilityDictContent, + const HeaderPolicy *const headerPolicy) + : mBuffer(buffer), mProbabilityDictContent(probabilityDictContent), + mHeaderPolicy(headerPolicy) {} ~Ver4PatriciaTrieNodeReader() {} @@ -50,6 +53,7 @@ class Ver4PatriciaTrieNodeReader : public PtNodeReader { const BufferWithExtendableBuffer *const mBuffer; const ProbabilityDictContent *const mProbabilityDictContent; + const HeaderPolicy *const mHeaderPolicy; const PtNodeParams fetchPtNodeInfoFromBufferAndProcessMovedPtNode(const int ptNodePos, const int siblingNodePos) const; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp index 32576cf0a..13ae9d923 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp @@ -159,7 +159,7 @@ bool Ver4PatriciaTrieNodeWriter::updatePtNodeProbabilityAndGetNeedsToKeepPtNodeA toBeUpdatedPtNodeParams->getTerminalId()); if (originalProbabilityEntry.hasHistoricalInfo()) { const HistoricalInfo historicalInfo = ForgettingCurveUtils::createHistoricalInfoToSave( - originalProbabilityEntry.getHistoricalInfo()); + originalProbabilityEntry.getHistoricalInfo(), mHeaderPolicy); const ProbabilityEntry probabilityEntry = originalProbabilityEntry.createEntryWithUpdatedHistoricalInfo(&historicalInfo); if (!mBuffers->getMutableProbabilityDictContent()->setProbabilityEntry( @@ -382,10 +382,11 @@ const ProbabilityEntry Ver4PatriciaTrieNodeWriter::createUpdatedEntryFrom( const ProbabilityEntry *const originalProbabilityEntry, const int newProbability, const int timestamp) const { // TODO: Consolidate historical info and probability. - if (mBuffers->getHeaderPolicy()->hasHistoricalInfoOfWords()) { + if (mHeaderPolicy->hasHistoricalInfoOfWords()) { const HistoricalInfo updatedHistoricalInfo = ForgettingCurveUtils::createUpdatedHistoricalInfo( - originalProbabilityEntry->getHistoricalInfo(), newProbability, timestamp); + originalProbabilityEntry->getHistoricalInfo(), newProbability, timestamp, + mHeaderPolicy); return originalProbabilityEntry->createEntryWithUpdatedHistoricalInfo( &updatedHistoricalInfo); } else { diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h index 66845bbd6..f01b3af0e 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h @@ -28,6 +28,7 @@ namespace latinime { class BufferWithExtendableBuffer; +class HeaderPolicy; class Ver4BigramListPolicy; class Ver4DictBuffers; class Ver4PatriciaTrieNodeReader; @@ -40,10 +41,11 @@ class Ver4ShortcutListPolicy; class Ver4PatriciaTrieNodeWriter : public PtNodeWriter { public: Ver4PatriciaTrieNodeWriter(BufferWithExtendableBuffer *const trieBuffer, - Ver4DictBuffers *const buffers, const PtNodeReader *const ptNodeReader, + Ver4DictBuffers *const buffers, const HeaderPolicy *const headerPolicy, + const PtNodeReader *const ptNodeReader, const PtNodeArrayReader *const ptNodeArrayReader, Ver4BigramListPolicy *const bigramPolicy, Ver4ShortcutListPolicy *const shortcutPolicy) - : mTrieBuffer(trieBuffer), mBuffers(buffers), + : mTrieBuffer(trieBuffer), mBuffers(buffers), mHeaderPolicy(headerPolicy), mReadingHelper(ptNodeReader, ptNodeArrayReader), mBigramPolicy(bigramPolicy), mShortcutPolicy(shortcutPolicy) {} @@ -116,6 +118,7 @@ class Ver4PatriciaTrieNodeWriter : public PtNodeWriter { BufferWithExtendableBuffer *const mTrieBuffer; Ver4DictBuffers *const mBuffers; + const HeaderPolicy *const mHeaderPolicy; DynamicPtReadingHelper mReadingHelper; Ver4BigramListPolicy *const mBigramPolicy; Ver4ShortcutListPolicy *const mShortcutPolicy; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp index b5d80be1d..197250ff3 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp @@ -382,7 +382,8 @@ const WordProperty Ver4PatriciaTriePolicy::getWordProperty(const int *const code bigramWord1CodePoints + codePointCount); const HistoricalInfo *const historicalInfo = bigramEntry.getHistoricalInfo(); const int probability = bigramEntry.hasHistoricalInfo() ? - ForgettingCurveUtils::decodeProbability(bigramEntry.getHistoricalInfo()) : + ForgettingCurveUtils::decodeProbability( + bigramEntry.getHistoricalInfo(), mHeaderPolicy) : bigramEntry.getProbability(); bigrams.push_back(WordProperty::BigramProperty(&word1, probability, historicalInfo->getTimeStamp(), historicalInfo->getLevel(), diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h index 7796e2ddc..639c153a1 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h @@ -47,10 +47,10 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { mBuffers.get()->getTerminalPositionLookupTable(), mHeaderPolicy), mShortcutPolicy(mBuffers.get()->getMutableShortcutDictContent(), mBuffers.get()->getTerminalPositionLookupTable()), - mNodeReader(mDictBuffer, mBuffers.get()->getProbabilityDictContent()), + mNodeReader(mDictBuffer, mBuffers.get()->getProbabilityDictContent(), mHeaderPolicy), mPtNodeArrayReader(mDictBuffer), - mNodeWriter(mDictBuffer, mBuffers.get(), &mNodeReader, &mPtNodeArrayReader, - &mBigramPolicy, &mShortcutPolicy), + mNodeWriter(mDictBuffer, mBuffers.get(), mHeaderPolicy, &mNodeReader, + &mPtNodeArrayReader, &mBigramPolicy, &mShortcutPolicy), mUpdatingHelper(mDictBuffer, &mNodeReader, &mNodeWriter), mWritingHelper(mBuffers.get()), mUnigramCount(mHeaderPolicy->getUnigramCount()), diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp index 93053c38d..2da295054 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp @@ -74,14 +74,15 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, const HeaderPolicy *const headerPolicy, Ver4DictBuffers *const buffersToWrite, int *const outUnigramCount, int *const outBigramCount) { Ver4PatriciaTrieNodeReader ptNodeReader(mBuffers->getTrieBuffer(), - mBuffers->getProbabilityDictContent()); + mBuffers->getProbabilityDictContent(), headerPolicy); Ver4PtNodeArrayReader ptNodeArrayReader(mBuffers->getTrieBuffer()); Ver4BigramListPolicy bigramPolicy(mBuffers->getMutableBigramDictContent(), mBuffers->getTerminalPositionLookupTable(), headerPolicy); Ver4ShortcutListPolicy shortcutPolicy(mBuffers->getMutableShortcutDictContent(), mBuffers->getTerminalPositionLookupTable()); Ver4PatriciaTrieNodeWriter ptNodeWriter(mBuffers->getWritableTrieBuffer(), - mBuffers, &ptNodeReader, &ptNodeArrayReader, &bigramPolicy, &shortcutPolicy); + mBuffers, headerPolicy, &ptNodeReader, &ptNodeArrayReader, &bigramPolicy, + &shortcutPolicy); DynamicPtReadingHelper readingHelper(&ptNodeReader, &ptNodeArrayReader); readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); @@ -126,7 +127,8 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, PtNodeWriter::DictPositionRelocationMap dictPositionRelocationMap; readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); Ver4PatriciaTrieNodeWriter ptNodeWriterForNewBuffers(buffersToWrite->getWritableTrieBuffer(), - buffersToWrite, &ptNodeReader, &ptNodeArrayReader, &bigramPolicy, &shortcutPolicy); + buffersToWrite, headerPolicy, &ptNodeReader, &ptNodeArrayReader, &bigramPolicy, + &shortcutPolicy); DynamicPtGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer traversePolicyToPlaceAndWriteValidPtNodesToBuffer(&ptNodeWriterForNewBuffers, buffersToWrite->getWritableTrieBuffer(), &dictPositionRelocationMap); @@ -137,14 +139,14 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, // Create policy instances for the GCed dictionary. Ver4PatriciaTrieNodeReader newPtNodeReader(buffersToWrite->getTrieBuffer(), - buffersToWrite->getProbabilityDictContent()); + buffersToWrite->getProbabilityDictContent(), headerPolicy); Ver4PtNodeArrayReader newPtNodeArrayreader(buffersToWrite->getTrieBuffer()); Ver4BigramListPolicy newBigramPolicy(buffersToWrite->getMutableBigramDictContent(), buffersToWrite->getTerminalPositionLookupTable(), headerPolicy); Ver4ShortcutListPolicy newShortcutPolicy(buffersToWrite->getMutableShortcutDictContent(), buffersToWrite->getTerminalPositionLookupTable()); Ver4PatriciaTrieNodeWriter newPtNodeWriter(buffersToWrite->getWritableTrieBuffer(), - buffersToWrite, &newPtNodeReader, &newPtNodeArrayreader, &newBigramPolicy, + buffersToWrite, headerPolicy, &newPtNodeReader, &newPtNodeArrayreader, &newBigramPolicy, &newShortcutPolicy); // Re-assign terminal IDs for valid terminal PtNodes. TerminalPositionLookupTable::TerminalIdMap terminalIdMap; @@ -202,8 +204,9 @@ bool Ver4PatriciaTrieWritingHelper::truncateUnigrams( const ProbabilityEntry probabilityEntry = mBuffers->getProbabilityDictContent()->getProbabilityEntry(i); const int probability = probabilityEntry.hasHistoricalInfo() ? - ForgettingCurveUtils::decodeProbability(probabilityEntry.getHistoricalInfo()) : - probabilityEntry.getProbability(); + ForgettingCurveUtils::decodeProbability( + probabilityEntry.getHistoricalInfo(), mBuffers->getHeaderPolicy()) : + probabilityEntry.getProbability(); priorityQueue.push(DictProbability(terminalPos, probability, probabilityEntry.getHistoricalInfo()->getTimeStamp())); } @@ -245,8 +248,9 @@ bool Ver4PatriciaTrieWritingHelper::truncateBigrams(const int maxBigramCount) { continue; } const int probability = bigramEntry.hasHistoricalInfo() ? - ForgettingCurveUtils::decodeProbability(bigramEntry.getHistoricalInfo()) : - bigramEntry.getProbability(); + ForgettingCurveUtils::decodeProbability( + bigramEntry.getHistoricalInfo(), mBuffers->getHeaderPolicy()) : + bigramEntry.getProbability(); priorityQueue.push(DictProbability(entryPos, probability, bigramEntry.getHistoricalInfo()->getTimeStamp())); } diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp index d58d25989..c9bb1093c 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp @@ -19,7 +19,7 @@ #include #include -#include "suggest/core/policy/dictionary_header_structure_policy.h" +#include "suggest/policyimpl/dictionary/header/header_policy.h" #include "suggest/policyimpl/dictionary/utils/probability_utils.h" #include "utils/time_keeper.h" @@ -34,7 +34,6 @@ const int ForgettingCurveUtils::MAX_COMPUTED_PROBABILITY = 127; const int ForgettingCurveUtils::DECAY_INTERVAL_SECONDS = 2 * 60 * 60; const int ForgettingCurveUtils::MAX_LEVEL = 3; -const int ForgettingCurveUtils::MAX_COUNT = 3; const int ForgettingCurveUtils::MIN_VALID_LEVEL = 1; const int ForgettingCurveUtils::TIME_STEP_DURATION_IN_SECONDS = 6 * 60 * 60; const int ForgettingCurveUtils::MAX_ELAPSED_TIME_STEP_COUNT = 15; @@ -45,7 +44,7 @@ const ForgettingCurveUtils::ProbabilityTable ForgettingCurveUtils::sProbabilityT // TODO: Revise the logic to decide the initial probability depending on the given probability. /* static */ const HistoricalInfo ForgettingCurveUtils::createUpdatedHistoricalInfo( const HistoricalInfo *const originalHistoricalInfo, - const int newProbability, const int timestamp) { + const int newProbability, const int timestamp, const HeaderPolicy *const headerPolicy) { if (newProbability != NOT_A_PROBABILITY && originalHistoricalInfo->getLevel() == 0) { return HistoricalInfo(timestamp, MIN_VALID_LEVEL /* level */, 0 /* count */); } else if (!originalHistoricalInfo->isValid()) { @@ -53,7 +52,7 @@ const ForgettingCurveUtils::ProbabilityTable ForgettingCurveUtils::sProbabilityT return HistoricalInfo(timestamp, 0 /* level */, 1 /* count */); } else { const int updatedCount = originalHistoricalInfo->getCount() + 1; - if (updatedCount > MAX_COUNT) { + if (updatedCount >= headerPolicy->getForgettingCurveOccurrencesToLevelUp()) { // The count exceeds the max value the level can be incremented. if (originalHistoricalInfo->getLevel() >= MAX_LEVEL) { // The level is already max. @@ -71,9 +70,10 @@ const ForgettingCurveUtils::ProbabilityTable ForgettingCurveUtils::sProbabilityT } /* static */ int ForgettingCurveUtils::decodeProbability( - const HistoricalInfo *const historicalInfo) { + const HistoricalInfo *const historicalInfo, const HeaderPolicy *const headerPolicy) { const int elapsedTimeStepCount = getElapsedTimeStepCount(historicalInfo->getTimeStamp()); - return sProbabilityTable.getProbability(historicalInfo->getLevel(), + return sProbabilityTable.getProbability( + headerPolicy->getForgettingCurveProbabilityValuesTableId(), historicalInfo->getLevel(), min(max(elapsedTimeStepCount, 0), MAX_ELAPSED_TIME_STEP_COUNT)); } @@ -95,7 +95,8 @@ const ForgettingCurveUtils::ProbabilityTable ForgettingCurveUtils::sProbabilityT } /* static */ const HistoricalInfo ForgettingCurveUtils::createHistoricalInfoToSave( - const HistoricalInfo *const originalHistoricalInfo) { + const HistoricalInfo *const originalHistoricalInfo, + const HeaderPolicy *const headerPolicy) { if (originalHistoricalInfo->getTimeStamp() == NOT_A_TIMESTAMP) { return HistoricalInfo(); } @@ -115,8 +116,7 @@ const ForgettingCurveUtils::ProbabilityTable ForgettingCurveUtils::sProbabilityT } /* static */ bool ForgettingCurveUtils::needsToDecay(const bool mindsBlockByDecay, - const int unigramCount, const int bigramCount, - const DictionaryHeaderStructurePolicy *const headerPolicy) { + const int unigramCount, const int bigramCount, const HeaderPolicy *const headerPolicy) { if (unigramCount >= ForgettingCurveUtils::MAX_UNIGRAM_COUNT) { // Unigram count exceeds the limit. return true; @@ -148,24 +148,30 @@ const ForgettingCurveUtils::ProbabilityTable ForgettingCurveUtils::sProbabilityT return (TimeKeeper::peekCurrentTime() - timestamp) / TIME_STEP_DURATION_IN_SECONDS; } -ForgettingCurveUtils::ProbabilityTable::ProbabilityTable() : mTable() { - mTable.resize(MAX_LEVEL + 1); - for (int level = 0; level <= MAX_LEVEL; ++level) { - mTable[level].resize(MAX_ELAPSED_TIME_STEP_COUNT + 1); - const float initialProbability = - static_cast(MAX_COMPUTED_PROBABILITY / (1 << (MAX_LEVEL - level))); - for (int timeStepCount = 0; timeStepCount <= MAX_ELAPSED_TIME_STEP_COUNT; ++timeStepCount) { - if (level == 0) { - mTable[level][timeStepCount] = NOT_A_PROBABILITY; - continue; +const int ForgettingCurveUtils::ProbabilityTable::PROBABILITY_TABLE_COUNT = 1; + +ForgettingCurveUtils::ProbabilityTable::ProbabilityTable() : mTables() { + mTables.resize(PROBABILITY_TABLE_COUNT); + for (int tableId = 0; tableId < PROBABILITY_TABLE_COUNT; ++tableId) { + mTables[tableId].resize(MAX_LEVEL + 1); + for (int level = 0; level <= MAX_LEVEL; ++level) { + mTables[tableId][level].resize(MAX_ELAPSED_TIME_STEP_COUNT + 1); + const float initialProbability = + static_cast(MAX_COMPUTED_PROBABILITY / (1 << (MAX_LEVEL - level))); + for (int timeStepCount = 0; timeStepCount <= MAX_ELAPSED_TIME_STEP_COUNT; + ++timeStepCount) { + if (level == 0) { + mTables[tableId][level][timeStepCount] = NOT_A_PROBABILITY; + continue; + } + const int elapsedTime = timeStepCount * TIME_STEP_DURATION_IN_SECONDS; + const float probability = initialProbability + * powf(2.0f, -1.0f * static_cast(elapsedTime) + / static_cast(TIME_STEP_DURATION_IN_SECONDS + * (MAX_ELAPSED_TIME_STEP_COUNT + 1))); + mTables[tableId][level][timeStepCount] = + min(max(static_cast(probability), 1), MAX_COMPUTED_PROBABILITY); } - const int elapsedTime = timeStepCount * TIME_STEP_DURATION_IN_SECONDS; - const float probability = initialProbability - * powf(2.0f, -1.0f * static_cast(elapsedTime) - / static_cast(TIME_STEP_DURATION_IN_SECONDS - * (MAX_ELAPSED_TIME_STEP_COUNT + 1))); - mTable[level][timeStepCount] = - min(max(static_cast(probability), 1), MAX_COMPUTED_PROBABILITY); } } } diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h index b37353455..db250474f 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h @@ -24,7 +24,7 @@ namespace latinime { -class DictionaryHeaderStructurePolicy; +class HeaderPolicy; class ForgettingCurveUtils { public: @@ -35,12 +35,14 @@ class ForgettingCurveUtils { static const HistoricalInfo createUpdatedHistoricalInfo( const HistoricalInfo *const originalHistoricalInfo, const int newProbability, - const int timestamp); + const int timestamp, const HeaderPolicy *const headerPolicy); static const HistoricalInfo createHistoricalInfoToSave( - const HistoricalInfo *const originalHistoricalInfo); + const HistoricalInfo *const originalHistoricalInfo, + const HeaderPolicy *const headerPolicy); - static int decodeProbability(const HistoricalInfo *const historicalInfo); + static int decodeProbability(const HistoricalInfo *const historicalInfo, + const HeaderPolicy *const headerPolicy); static int getProbability(const int encodedUnigramProbability, const int encodedBigramProbability); @@ -48,7 +50,7 @@ class ForgettingCurveUtils { static bool needsToKeep(const HistoricalInfo *const historicalInfo); static bool needsToDecay(const bool mindsBlockByDecay, const int unigramCount, - const int bigramCount, const DictionaryHeaderStructurePolicy *const headerPolicy); + const int bigramCount, const HeaderPolicy *const headerPolicy); private: DISALLOW_IMPLICIT_CONSTRUCTORS(ForgettingCurveUtils); @@ -57,21 +59,23 @@ class ForgettingCurveUtils { public: ProbabilityTable(); - int getProbability(const int level, const int elapsedTimeStepCount) const { - return mTable[level][elapsedTimeStepCount]; + int getProbability(const int tableId, const int level, + const int elapsedTimeStepCount) const { + return mTables[tableId][level][elapsedTimeStepCount]; } private: DISALLOW_COPY_AND_ASSIGN(ProbabilityTable); - std::vector > mTable; + static const int PROBABILITY_TABLE_COUNT; + + std::vector > > mTables; }; static const int MAX_COMPUTED_PROBABILITY; static const int DECAY_INTERVAL_SECONDS; static const int MAX_LEVEL; - static const int MAX_COUNT; static const int MIN_VALID_LEVEL; static const int TIME_STEP_DURATION_IN_SECONDS; static const int MAX_ELAPSED_TIME_STEP_COUNT;