Merge "Implement Ver4PatriciaTriePolicy::needsToRunGC()."

main
Keisuke Kuroyanagi 2013-11-21 08:40:25 +00:00 committed by Android (Google) Code Review
commit 9882fbb2ee
7 changed files with 43 additions and 3 deletions

View File

@ -46,6 +46,10 @@ class SingleDictContent : public DictContent {
return mIsValid; return mIsValid;
} }
bool isNearSizeLimit() const {
return mExpandableContentBuffer.isNearSizeLimit();
}
protected: protected:
BufferWithExtendableBuffer *getWritableBuffer() { BufferWithExtendableBuffer *getWritableBuffer() {
return &mExpandableContentBuffer; return &mExpandableContentBuffer;

View File

@ -69,6 +69,12 @@ class SparseTableDictContent : public DictContent {
return mIsValid; return mIsValid;
} }
bool isNearSizeLimit() const {
return mExpandableLookupTableBuffer.isNearSizeLimit()
|| mExpandableAddressTableBuffer.isNearSizeLimit()
|| mExpandableContentBuffer.isNearSizeLimit();
}
protected: protected:
SparseTable *getUpdatableAddressLookupTable() { SparseTable *getUpdatableAddressLookupTable() {
return &mAddressLookupTable; return &mAddressLookupTable;

View File

@ -49,6 +49,14 @@ class Ver4DictBuffers {
&& mShortcutDictContent.isValid(); && mShortcutDictContent.isValid();
} }
AK_FORCE_INLINE bool isNearSizeLimit() const {
return mExpandableTrieBuffer.isNearSizeLimit()
|| mTerminalPositionLookupTable.isNearSizeLimit()
|| mProbabilityDictContent.isNearSizeLimit()
|| mBigramDictContent.isNearSizeLimit()
|| mShortcutDictContent.isNearSizeLimit();
}
AK_FORCE_INLINE BufferWithExtendableBuffer *getWritableHeaderBuffer() { AK_FORCE_INLINE BufferWithExtendableBuffer *getWritableHeaderBuffer() {
return &mExpandableHeaderBuffer; return &mExpandableHeaderBuffer;
} }

View File

@ -33,6 +33,9 @@ const char *const Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION =
// Version 4 dictionary size is implicitly limited to 8MB due to 3-byte offsets. // Version 4 dictionary size is implicitly limited to 8MB due to 3-byte offsets.
// TODO: Make MAX_DICTIONARY_SIZE 8MB. // TODO: Make MAX_DICTIONARY_SIZE 8MB.
const int Ver4DictConstants::MAX_DICTIONARY_SIZE = 2 * 1024 * 1024; const int Ver4DictConstants::MAX_DICTIONARY_SIZE = 2 * 1024 * 1024;
// Extended region size, which is not GCed region size in dict file + additional buffer size, is
// limited to 1MB to prevent from inefficient traversing.
const int Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE = 1 * 1024 * 1024;
const int Ver4DictConstants::NOT_A_TERMINAL_ID = -1; const int Ver4DictConstants::NOT_A_TERMINAL_ID = -1;
const int Ver4DictConstants::PROBABILITY_SIZE = 1; const int Ver4DictConstants::PROBABILITY_SIZE = 1;

View File

@ -35,6 +35,7 @@ class Ver4DictConstants {
static const char *const SHORTCUT_CONTENT_TABLE_FILE_EXTENSION; static const char *const SHORTCUT_CONTENT_TABLE_FILE_EXTENSION;
static const int MAX_DICTIONARY_SIZE; static const int MAX_DICTIONARY_SIZE;
static const int MAX_DICT_EXTENDED_REGION_SIZE;
static const int NOT_A_TERMINAL_ID; static const int NOT_A_TERMINAL_ID;
static const int PROBABILITY_SIZE; static const int PROBABILITY_SIZE;

View File

@ -19,7 +19,6 @@
#include "suggest/core/dicnode/dic_node.h" #include "suggest/core/dicnode/dic_node.h"
#include "suggest/core/dicnode/dic_node_vector.h" #include "suggest/core/dicnode/dic_node_vector.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h" #include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_helper.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h" #include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
#include "suggest/policyimpl/dictionary/utils/probability_utils.h" #include "suggest/policyimpl/dictionary/utils/probability_utils.h"
@ -228,7 +227,25 @@ void Ver4PatriciaTriePolicy::flushWithGC(const char *const filePath) {
} }
bool Ver4PatriciaTriePolicy::needsToRunGC(const bool mindsBlockByGC) const { bool Ver4PatriciaTriePolicy::needsToRunGC(const bool mindsBlockByGC) const {
// TODO: Implement. if (!mBuffers.get()->isUpdatable()) {
AKLOGI("Warning: needsToRunGC() is called for non-updatable dictionary.");
return false;
}
if (mBuffers.get()->isNearSizeLimit()) {
// Additional buffer size is near the limit.
return true;
} else if (mHeaderPolicy.getExtendedRegionSize() + mDictBuffer->getUsedAdditionalBufferSize()
> Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE) {
// Total extended region size of the trie exceeds the limit.
return true;
} else if (mDictBuffer->getTailPosition() >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS
&& mDictBuffer->getUsedAdditionalBufferSize() > 0) {
// Needs to reduce dictionary size.
return true;
} else if (mHeaderPolicy.isDecayingDict()) {
return mNeedsToDecayForTesting || ForgettingCurveUtils::needsToDecay(
mindsBlockByGC, mUnigramCount, mBigramCount, &mHeaderPolicy);
}
return false; return false;
} }

View File

@ -53,7 +53,7 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
mHeaderPolicy.isDecayingDict()), mHeaderPolicy.isDecayingDict()),
mWritingHelper(mBuffers.get()), mWritingHelper(mBuffers.get()),
mUnigramCount(mHeaderPolicy.getUnigramCount()), mUnigramCount(mHeaderPolicy.getUnigramCount()),
mBigramCount(mHeaderPolicy.getBigramCount()) {}; mBigramCount(mHeaderPolicy.getBigramCount()), mNeedsToDecayForTesting(false) {};
AK_FORCE_INLINE int getRootPosition() const { AK_FORCE_INLINE int getRootPosition() const {
return 0; return 0;
@ -125,6 +125,7 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
Ver4PatriciaTrieWritingHelper mWritingHelper; Ver4PatriciaTrieWritingHelper mWritingHelper;
int mUnigramCount; int mUnigramCount;
int mBigramCount; int mBigramCount;
bool mNeedsToDecayForTesting;
}; };
} // namespace latinime } // namespace latinime
#endif // LATINIME_VER4_PATRICIA_TRIE_POLICY_H #endif // LATINIME_VER4_PATRICIA_TRIE_POLICY_H