nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

am 1b0e5748: Refactoring: Make GC listeners version independent. 

* commit '1b0e574830d51ce119fd22d8a472ca634c42f2f3':
  Refactoring: Make GC listeners version independent.
main
Keisuke Kuroyanagi 2013-11-21 04:21:42 -08:00 committed by Android Git Automerger
parent 7196fcd516 1b0e574830
commit 5b79a2222d
13 changed files with 198 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp
View File

@ -182,8 +182,8 @@ bool DynamicBigramListPolicy::updateAllBigramEntriesAndDeleteUselessEntries(
// Updates bigram target PtNode positions in the list after the placing step in GC.
bool DynamicBigramListPolicy::updateAllBigramTargetPtNodePositions(int *const bigramListPos,
const DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap *const
ptNodePositionRelocationMap, int *const outBigramEntryCount) {
const PtNodeWriter::PtNodePositionRelocationMap *const ptNodePositionRelocationMap,
int *const outBigramEntryCount) {
const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*bigramListPos);
if (usesAdditionalBuffer) {
*bigramListPos -= mBuffer->getOriginalBufferSize();
@ -213,7 +213,7 @@ bool DynamicBigramListPolicy::updateAllBigramTargetPtNodePositions(int *const bi
bigramTargetPtNodePos += mBuffer->getOriginalBufferSize();
}
DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap::const_iterator it =
PtNodeWriter::PtNodePositionRelocationMap::const_iterator it =
ptNodePositionRelocationMap->find(bigramTargetPtNodePos);
if (it != ptNodePositionRelocationMap->end()) {
bigramTargetPtNodePos = it->second;

5
native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h
View File

@ -22,7 +22,8 @@
#include "defines.h"
#include "suggest/core/policy/dictionary_bigrams_structure_policy.h"
#include "suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_helper.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h"
namespace latinime {
@ -59,7 +60,7 @@ class DynamicBigramListPolicy : public DictionaryBigramsStructurePolicy {
int *const outBigramEntryCount);
bool updateAllBigramTargetPtNodePositions(int *const bigramListPos,
const DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap *const
const PtNodeWriter::PtNodePositionRelocationMap *const
ptNodePositionRelocationMap, int *const outValidBigramEntryCount);
bool addNewBigramEntryToBigramList(const int bigramTargetPos, const int probability,

22
native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h
View File

@ -20,12 +20,27 @@
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h"
#include "utils/hash_map_compat.h"
namespace latinime {
// Interface class used to write PtNode information.
class PtNodeWriter {
public:
typedef hash_map_compat<int, int> PtNodeArrayPositionRelocationMap;
typedef hash_map_compat<int, int> PtNodePositionRelocationMap;
struct DictPositionRelocationMap {
public:
DictPositionRelocationMap()
: mPtNodeArrayPositionRelocationMap(), mPtNodePositionRelocationMap() {}
PtNodeArrayPositionRelocationMap mPtNodeArrayPositionRelocationMap;
PtNodePositionRelocationMap mPtNodePositionRelocationMap;
private:
DISALLOW_COPY_AND_ASSIGN(DictPositionRelocationMap);
};
virtual ~PtNodeWriter() {}
virtual bool markPtNodeAsDeleted(const PtNodeParams *const toBeUpdatedPtNodeParams) = 0;
@ -49,6 +64,13 @@ class PtNodeWriter {
virtual bool removeBigramEntry(const PtNodeParams *const sourcePtNodeParams,
const PtNodeParams *const targetPtNodeParam) = 0;
virtual bool updateAllBigramEntriesAndDeleteUselessEntries(
const PtNodeParams *const sourcePtNodeParams, int *const outBigramEntryCount) = 0;
virtual bool updateAllPositionFields(const PtNodeParams *const toBeUpdatedPtNodeParams,
const DictPositionRelocationMap *const dictPositionRelocationMap,
int *const outBigramEntryCount) = 0;
protected:
PtNodeWriter() {};

62
native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.cpp
View File

@ -34,10 +34,8 @@ bool DynamicPatriciaTrieGcEventListeners
const int newProbability =
ForgettingCurveUtils::getEncodedProbabilityToSave(ptNodeParams->getProbability(),
mHeaderPolicy);
int writingPos = ptNodeParams->getProbabilityFieldPos();
// Update probability.
if (!DynamicPatriciaTrieWritingUtils::writeProbabilityAndAdvancePosition(
mBuffer, newProbability, &writingPos)) {
if (!mPtNodeWriter->updatePtNodeProbability(ptNodeParams, newProbability)) {
return false;
}
if (!ForgettingCurveUtils::isValidEncodedProbability(newProbability)) {
@ -48,9 +46,8 @@ bool DynamicPatriciaTrieGcEventListeners
isUselessPtNode = false;
} else if (ptNodeParams->isTerminal()) {
// Remove children as all children are useless.
int writingPos = ptNodeParams->getChildrenPosFieldPos();
if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(
mBuffer, NOT_A_DICT_POS /* childrenPosition */, &writingPos)) {
if (!mPtNodeWriter->updateChildrenPosition(ptNodeParams,
NOT_A_DICT_POS /* newChildrenPosition */)) {
return false;
}
}
@ -74,7 +71,7 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateBigramProbabilit
int pos = ptNodeParams->getBigramsPos();
if (pos != NOT_A_DICT_POS) {
int bigramEntryCount = 0;
if (!mBigramPolicy->updateAllBigramEntriesAndDeleteUselessEntries(&pos,
if (!mPtNodeWriter->updateAllBigramEntriesAndDeleteUselessEntries(ptNodeParams,
&bigramEntryCount)) {
return false;
}
@ -90,8 +87,7 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNo
mValidPtNodeCount = 0;
int writingPos = mBufferToWrite->getTailPosition();
mDictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.insert(
DynamicPatriciaTrieWritingHelper::PtNodeArrayPositionRelocationMap::value_type(
ptNodeArrayPos, writingPos));
PtNodeWriter::PtNodeArrayPositionRelocationMap::value_type(ptNodeArrayPos, writingPos));
// Writes dummy PtNode array size because arrays can have a forward link or needles PtNodes.
// This field will be updated later in onReadingPtNodeArrayTail() with actual PtNode count.
mPtNodeArraySizeFieldPos = writingPos;
@ -122,13 +118,13 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNo
if (ptNodeParams->isDeleted()) {
// Current PtNode is not written in new buffer because it has been deleted.
mDictPositionRelocationMap->mPtNodePositionRelocationMap.insert(
DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap::value_type(
PtNodeWriter::PtNodePositionRelocationMap::value_type(
ptNodeParams->getHeadPos(), NOT_A_DICT_POS));
return true;
}
int writingPos = mBufferToWrite->getTailPosition();
mDictPositionRelocationMap->mPtNodePositionRelocationMap.insert(
DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap::value_type(
PtNodeWriter::PtNodePositionRelocationMap::value_type(
ptNodeParams->getHeadPos(), writingPos));
mValidPtNodeCount++;
// Writes current PtNode.
@ -138,51 +134,15 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNo
bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateAllPositionFields
::onVisitingPtNode(const PtNodeParams *const ptNodeParams) {
// Updates parent position.
int parentPos = ptNodeParams->getParentPos();
if (parentPos != NOT_A_DICT_POS) {
DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap::const_iterator it =
mDictPositionRelocationMap->mPtNodePositionRelocationMap.find(parentPos);
if (it != mDictPositionRelocationMap->mPtNodePositionRelocationMap.end()) {
parentPos = it->second;
}
}
int writingPos = ptNodeParams->getHeadPos()
+ DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE;
// Write updated parent offset.
if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition(mBufferToWrite,
parentPos, ptNodeParams->getHeadPos(), &writingPos)) {
int bigramCount = 0;
if (!mPtNodeWriter->updateAllPositionFields(ptNodeParams, mDictPositionRelocationMap,
&bigramCount)) {
return false;
}
// Updates children position.
int childrenPos = ptNodeParams->getChildrenPos();
if (childrenPos != NOT_A_DICT_POS) {
DynamicPatriciaTrieWritingHelper::PtNodeArrayPositionRelocationMap::const_iterator it =
mDictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.find(childrenPos);
if (it != mDictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.end()) {
childrenPos = it->second;
}
}
writingPos = ptNodeParams->getChildrenPosFieldPos();
if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(mBufferToWrite,
childrenPos, &writingPos)) {
return false;
}
// Updates bigram target PtNode positions in the bigram list.
int bigramsPos = ptNodeParams->getBigramsPos();
if (bigramsPos != NOT_A_DICT_POS) {
int bigramEntryCount;
if (!mBigramPolicy->updateAllBigramTargetPtNodePositions(&bigramsPos,
&mDictPositionRelocationMap->mPtNodePositionRelocationMap, &bigramEntryCount)) {
return false;
}
mBigramCount += bigramEntryCount;
}
mBigramCount += bigramCount;
if (ptNodeParams->isTerminal()) {
mUnigramCount++;
}
return true;
}

32
native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h
View File

@ -20,9 +20,8 @@
#include <vector>
#include "defines.h"
#include "suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_helper.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "utils/hash_map_compat.h"
@ -32,6 +31,7 @@ class DictionaryHeaderStructurePolicy;
class PtNodeWriter;
class PtNodeParams;
// TODO: Move to pt_common.
class DynamicPatriciaTrieGcEventListeners {
public:
// Updates all PtNodes that can be reached from the root. Checks if each PtNode is useless or
@ -91,9 +91,8 @@ class DynamicPatriciaTrieGcEventListeners {
class TraversePolicyToUpdateBigramProbability
: public DynamicPatriciaTrieReadingHelper::TraversingEventListener {
public:
TraversePolicyToUpdateBigramProbability(
DynamicBigramListPolicy *const bigramPolicy)
: mBigramPolicy(bigramPolicy), mValidBigramEntryCount(0) {}
TraversePolicyToUpdateBigramProbability(PtNodeWriter *const ptNodeWriter)
: mPtNodeWriter(ptNodeWriter), mValidBigramEntryCount(0) {}
bool onAscend() { return true; }
@ -110,7 +109,7 @@ class DynamicPatriciaTrieGcEventListeners {
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(TraversePolicyToUpdateBigramProbability);
DynamicBigramListPolicy *const mBigramPolicy;
PtNodeWriter *const mPtNodeWriter;
int mValidBigramEntryCount;
};
@ -119,8 +118,7 @@ class DynamicPatriciaTrieGcEventListeners {
public:
TraversePolicyToPlaceAndWriteValidPtNodesToBuffer(
PtNodeWriter *const ptNodeWriter, BufferWithExtendableBuffer *const bufferToWrite,
DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const
dictPositionRelocationMap)
PtNodeWriter::DictPositionRelocationMap *const dictPositionRelocationMap)
: mPtNodeWriter(ptNodeWriter), mBufferToWrite(bufferToWrite),
mDictPositionRelocationMap(dictPositionRelocationMap), mValidPtNodeCount(0),
mPtNodeArraySizeFieldPos(NOT_A_DICT_POS) {};
@ -138,8 +136,7 @@ class DynamicPatriciaTrieGcEventListeners {
PtNodeWriter *const mPtNodeWriter;
BufferWithExtendableBuffer *const mBufferToWrite;
DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const
mDictPositionRelocationMap;
PtNodeWriter::DictPositionRelocationMap *const mDictPositionRelocationMap;
int mValidPtNodeCount;
int mPtNodeArraySizeFieldPos;
};
@ -147,12 +144,9 @@ class DynamicPatriciaTrieGcEventListeners {
class TraversePolicyToUpdateAllPositionFields
: public DynamicPatriciaTrieReadingHelper::TraversingEventListener {
public:
TraversePolicyToUpdateAllPositionFields(
DynamicBigramListPolicy *const bigramPolicy,
BufferWithExtendableBuffer *const bufferToWrite,
const DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const
dictPositionRelocationMap)
: mBigramPolicy(bigramPolicy), mBufferToWrite(bufferToWrite),
TraversePolicyToUpdateAllPositionFields(PtNodeWriter *const ptNodeWriter,
const PtNodeWriter::DictPositionRelocationMap *const dictPositionRelocationMap)
: mPtNodeWriter(ptNodeWriter),
mDictPositionRelocationMap(dictPositionRelocationMap), mUnigramCount(0),
mBigramCount(0) {};
@ -175,10 +169,8 @@ class DynamicPatriciaTrieGcEventListeners {
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(TraversePolicyToUpdateAllPositionFields);
DynamicBigramListPolicy *const mBigramPolicy;
BufferWithExtendableBuffer *const mBufferToWrite;
const DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const
mDictPositionRelocationMap;
PtNodeWriter *const mPtNodeWriter;
const PtNodeWriter::DictPositionRelocationMap *const mDictPositionRelocationMap;
int mUnigramCount;
int mBigramCount;
};

56
native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_node_writer.cpp
View File

@ -233,4 +233,60 @@ bool DynamicPatriciaTrieNodeWriter::removeBigramEntry(
targetPtNodeParam->getHeadPos());
}
bool DynamicPatriciaTrieNodeWriter::updateAllBigramEntriesAndDeleteUselessEntries(
const PtNodeParams *const sourcePtNodeParams, int *const outBigramEntryCount) {
int bigramListPos = sourcePtNodeParams->getBigramsPos();
return mBigramPolicy->updateAllBigramEntriesAndDeleteUselessEntries(&bigramListPos,
outBigramEntryCount);
}
bool DynamicPatriciaTrieNodeWriter::updateAllPositionFields(
const PtNodeParams *const toBeUpdatedPtNodeParams,
const DictPositionRelocationMap *const dictPositionRelocationMap,
int *const outBigramEntryCount) {
int parentPos = toBeUpdatedPtNodeParams->getParentPos();
if (parentPos != NOT_A_DICT_POS) {
PtNodeWriter::PtNodePositionRelocationMap::const_iterator it =
dictPositionRelocationMap->mPtNodePositionRelocationMap.find(parentPos);
if (it != dictPositionRelocationMap->mPtNodePositionRelocationMap.end()) {
parentPos = it->second;
}
}
int writingPos = toBeUpdatedPtNodeParams->getHeadPos()
+ DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE;
// Write an updated offset to the parent PtNode.
if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition(mBuffer,
parentPos, toBeUpdatedPtNodeParams->getHeadPos(), &writingPos)) {
return false;
}
// Updates children position that is a PtNodeArray position.
int childrenPos = toBeUpdatedPtNodeParams->getChildrenPos();
if (childrenPos != NOT_A_DICT_POS) {
PtNodeWriter::PtNodeArrayPositionRelocationMap::const_iterator it =
dictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.find(childrenPos);
if (it != dictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.end()) {
childrenPos = it->second;
}
}
if (!updateChildrenPosition(toBeUpdatedPtNodeParams, childrenPos)) {
return false;
}
// Updates bigram target PtNode positions in the bigram list.
int bigramsPos = toBeUpdatedPtNodeParams->getBigramsPos();
int bigramCount = 0;
if (bigramsPos != NOT_A_DICT_POS) {
int bigramEntryCount;
if (!mBigramPolicy->updateAllBigramTargetPtNodePositions(&bigramsPos,
&dictPositionRelocationMap->mPtNodePositionRelocationMap, &bigramEntryCount)) {
return false;
}
bigramCount += bigramEntryCount;
}
if (outBigramEntryCount) {
*outBigramEntryCount = bigramCount;
}
return true;
}
}

7
native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_node_writer.h
View File

@ -66,6 +66,13 @@ class DynamicPatriciaTrieNodeWriter : public PtNodeWriter {
virtual bool removeBigramEntry(const PtNodeParams *const sourcePtNodeParams,
const PtNodeParams *const targetPtNodeParam);
virtual bool updateAllBigramEntriesAndDeleteUselessEntries(
const PtNodeParams *const sourcePtNodeParams, int *const outBigramEntryCount);
virtual bool updateAllPositionFields(const PtNodeParams *const toBeUpdatedPtNodeParams,
const DictPositionRelocationMap *const dictPositionRelocationMap,
int *const outBigramEntryCount);
private:
DISALLOW_COPY_AND_ASSIGN(DynamicPatriciaTrieNodeWriter);

13
native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_helper.cpp
View File

@ -92,7 +92,7 @@ bool DynamicPatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos);
DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateBigramProbability
traversePolicyToUpdateBigramProbability(mBigramPolicy);
traversePolicyToUpdateBigramProbability(mPtNodeWriter);
if (!readingHelper.traverseAllPtNodesInPostorderDepthFirstManner(
&traversePolicyToUpdateBigramProbability)) {
return false;
@ -103,10 +103,10 @@ bool DynamicPatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
}
// Mapping from positions in mBuffer to positions in bufferToWrite.
DictPositionRelocationMap dictPositionRelocationMap;
PtNodeWriter::DictPositionRelocationMap dictPositionRelocationMap;
readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos);
DynamicPatriciaTrieNodeWriter newPtNodeWriter(bufferToWrite,
&ptNodeReader, mBigramPolicy, mShortcutPolicy);
DynamicPatriciaTrieNodeWriter newPtNodeWriter(bufferToWrite, &ptNodeReader, mBigramPolicy,
mShortcutPolicy);
DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer
traversePolicyToPlaceAndWriteValidPtNodesToBuffer(&newPtNodeWriter, bufferToWrite,
&dictPositionRelocationMap);
@ -123,10 +123,11 @@ bool DynamicPatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
DynamicPatriciaTrieNodeReader newDictNodeReader(bufferToWrite, &newDictBigramPolicy,
&newDictShortcutPolicy);
DynamicPatriciaTrieReadingHelper newDictReadingHelper(bufferToWrite, &newDictNodeReader);
DynamicPatriciaTrieNodeWriter newDictNodeWriter(bufferToWrite, &newDictNodeReader,
&newDictBigramPolicy, &newDictShortcutPolicy);
newDictReadingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos);
DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateAllPositionFields
traversePolicyToUpdateAllPositionFields(&newDictBigramPolicy, bufferToWrite,
&dictPositionRelocationMap);
traversePolicyToUpdateAllPositionFields(&newDictNodeWriter, &dictPositionRelocationMap);
if (!newDictReadingHelper.traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner(
&traversePolicyToUpdateAllPositionFields)) {
return false;

15
native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_helper.h
View File

@ -20,7 +20,6 @@
#include <stdint.h>
#include "defines.h"
#include "utils/hash_map_compat.h"
namespace latinime {
@ -35,20 +34,6 @@ class PtNodeWriter;
// TODO: Make it independent from a particular format and move to pt_common.
class DynamicPatriciaTrieWritingHelper {
public:
typedef hash_map_compat<int, int> PtNodeArrayPositionRelocationMap;
typedef hash_map_compat<int, int> PtNodePositionRelocationMap;
struct DictPositionRelocationMap {
public:
DictPositionRelocationMap()
: mPtNodeArrayPositionRelocationMap(), mPtNodePositionRelocationMap() {}
PtNodeArrayPositionRelocationMap mPtNodeArrayPositionRelocationMap;
PtNodePositionRelocationMap mPtNodePositionRelocationMap;
private:
DISALLOW_COPY_AND_ASSIGN(DictPositionRelocationMap);
};
static const size_t MAX_DICTIONARY_SIZE;
DynamicPatriciaTrieWritingHelper(BufferWithExtendableBuffer *const buffer,

14
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp
View File

@ -196,4 +196,18 @@ bool Ver4PatriciaTrieNodeWriter::removeBigramEntry(
targetPtNodeParam->getTerminalId());
}
bool Ver4PatriciaTrieNodeWriter::updateAllBigramEntriesAndDeleteUselessEntries(
const PtNodeParams *const sourcePtNodeParams, int *const outBigramEntryCount) {
// TODO: Implement.
return false;
}
bool Ver4PatriciaTrieNodeWriter::updateAllPositionFields(
const PtNodeParams *const toBeUpdatedPtNodeParams,
const DictPositionRelocationMap *const dictPositionRelocationMap,
int *const outBigramEntryCount) {
// TODO: Implement.
return false;
}
}

8
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h
View File

@ -67,6 +67,13 @@ class Ver4PatriciaTrieNodeWriter : public PtNodeWriter {
virtual bool removeBigramEntry(const PtNodeParams *const sourcePtNodeParams,
const PtNodeParams *const targetPtNodeParam);
virtual bool updateAllBigramEntriesAndDeleteUselessEntries(
const PtNodeParams *const sourcePtNodeParams, int *const outBigramEntryCount);
virtual bool updateAllPositionFields(const PtNodeParams *const toBeUpdatedPtNodeParams,
const DictPositionRelocationMap *const dictPositionRelocationMap,
int *const outBigramEntryCount);
private:
DISALLOW_COPY_AND_ASSIGN(Ver4PatriciaTrieNodeWriter);
@ -78,7 +85,6 @@ class Ver4PatriciaTrieNodeWriter : public PtNodeWriter {
DynamicPatriciaTrieReadingHelper mReadingHelper;
Ver4BigramListPolicy *const mBigramPolicy;
Ver4ShortcutListPolicy *const mShortcutPolicy;
};
} // namespace latinime
#endif /* LATINIME_VER4_PATRICIA_TRIE_NODE_WRITER_H */

48
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp
View File

@ -18,11 +18,16 @@
#include <cstring>
#include "suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/shortcut/ver4_shortcut_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/file_utils.h"
#include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
namespace latinime {
@ -47,4 +52,47 @@ void Ver4PatriciaTrieWritingHelper::writeToDictFile(const char *const trieFilePa
mBuffers->flushHeaderAndDictBuffers(dirPath, &headerBuffer);
}
void Ver4PatriciaTrieWritingHelper::writeToDictFileWithGC(const int rootPtNodeArrayPos,
const char *const trieFilePath, const HeaderPolicy *const headerPolicy,
const bool needsToDecay) {
Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers(Ver4DictBuffers::createVer4DictBuffers());
int unigramCount = 0;
int bigramCount = 0;
if (needsToDecay) {
ForgettingCurveUtils::sTimeKeeper.setCurrentTime();
}
if (!runGC(rootPtNodeArrayPos, headerPolicy, dictBuffers.get(), &unigramCount, &bigramCount,
needsToDecay)) {
return;
}
BufferWithExtendableBuffer headerBuffer(
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE);
if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, true /* updatesLastUpdatedTime */,
needsToDecay, unigramCount, bigramCount, 0 /* extendedRegionSize */)) {
return;
}
const int dirPathBufSize = strlen(trieFilePath) + 1 /* terminator */;
char dirPath[dirPathBufSize];
FileUtils::getDirPath(trieFilePath, dirPathBufSize, dirPath);
dictBuffers.get()->flushHeaderAndDictBuffers(dirPath, &headerBuffer);
}
bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
const HeaderPolicy *const headerPolicy, Ver4DictBuffers *const buffersToWrite,
int *const outUnigramCount, int *const outBigramCount, const bool needsToDecay) {
Ver4PatriciaTrieNodeReader ptNodeReader(mBuffers->getTrieBuffer(),
mBuffers->getProbabilityDictContent());
Ver4BigramListPolicy bigramPolicy(mBuffers->getUpdatableBigramDictContent(),
mBuffers->getTerminalPositionLookupTable());
Ver4ShortcutListPolicy shortcutPolicy(mBuffers->getShortcutDictContent(),
mBuffers->getTerminalPositionLookupTable());
Ver4PatriciaTrieNodeWriter ptNodeWriter(mBuffers->getWritableTrieBuffer(),
mBuffers, &ptNodeReader, &bigramPolicy, &shortcutPolicy);
DynamicPatriciaTrieReadingHelper readingHelper(mBuffers->getTrieBuffer(), &ptNodeReader);
readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos);
return true;
}
} // namespace latinime

8
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h
View File

@ -32,9 +32,17 @@ class Ver4PatriciaTrieWritingHelper {
void writeToDictFile(const char *const trieFilePath, const HeaderPolicy *const headerPolicy,
const int unigramCount, const int bigramCount) const;
void writeToDictFileWithGC(const int rootPtNodeArrayPos,
const char *const trieFilePath, const HeaderPolicy *const headerPolicy,
const bool needsToDecay);
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4PatriciaTrieWritingHelper);
bool runGC(const int rootPtNodeArrayPos, const HeaderPolicy *const headerPolicy,
Ver4DictBuffers *const buffersToWrite, int *const outUnigramCount,
int *const outBigramCount, const bool needsToDecay);
Ver4DictBuffers *const mBuffers;
};
} // namespace latinime