am 1b0e5748: Refactoring: Make GC listeners version independent.

* commit '1b0e574830d51ce119fd22d8a472ca634c42f2f3':
  Refactoring: Make GC listeners version independent.
main
Keisuke Kuroyanagi 2013-11-21 04:21:42 -08:00 committed by Android Git Automerger
commit 5b79a2222d
13 changed files with 198 additions and 98 deletions

View File

@ -182,8 +182,8 @@ bool DynamicBigramListPolicy::updateAllBigramEntriesAndDeleteUselessEntries(
// Updates bigram target PtNode positions in the list after the placing step in GC. // Updates bigram target PtNode positions in the list after the placing step in GC.
bool DynamicBigramListPolicy::updateAllBigramTargetPtNodePositions(int *const bigramListPos, bool DynamicBigramListPolicy::updateAllBigramTargetPtNodePositions(int *const bigramListPos,
const DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap *const const PtNodeWriter::PtNodePositionRelocationMap *const ptNodePositionRelocationMap,
ptNodePositionRelocationMap, int *const outBigramEntryCount) { int *const outBigramEntryCount) {
const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*bigramListPos); const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*bigramListPos);
if (usesAdditionalBuffer) { if (usesAdditionalBuffer) {
*bigramListPos -= mBuffer->getOriginalBufferSize(); *bigramListPos -= mBuffer->getOriginalBufferSize();
@ -213,7 +213,7 @@ bool DynamicBigramListPolicy::updateAllBigramTargetPtNodePositions(int *const bi
bigramTargetPtNodePos += mBuffer->getOriginalBufferSize(); bigramTargetPtNodePos += mBuffer->getOriginalBufferSize();
} }
DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap::const_iterator it = PtNodeWriter::PtNodePositionRelocationMap::const_iterator it =
ptNodePositionRelocationMap->find(bigramTargetPtNodePos); ptNodePositionRelocationMap->find(bigramTargetPtNodePos);
if (it != ptNodePositionRelocationMap->end()) { if (it != ptNodePositionRelocationMap->end()) {
bigramTargetPtNodePos = it->second; bigramTargetPtNodePos = it->second;

View File

@ -22,7 +22,8 @@
#include "defines.h" #include "defines.h"
#include "suggest/core/policy/dictionary_bigrams_structure_policy.h" #include "suggest/core/policy/dictionary_bigrams_structure_policy.h"
#include "suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h" #include "suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_helper.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h"
namespace latinime { namespace latinime {
@ -59,7 +60,7 @@ class DynamicBigramListPolicy : public DictionaryBigramsStructurePolicy {
int *const outBigramEntryCount); int *const outBigramEntryCount);
bool updateAllBigramTargetPtNodePositions(int *const bigramListPos, bool updateAllBigramTargetPtNodePositions(int *const bigramListPos,
const DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap *const const PtNodeWriter::PtNodePositionRelocationMap *const
ptNodePositionRelocationMap, int *const outValidBigramEntryCount); ptNodePositionRelocationMap, int *const outValidBigramEntryCount);
bool addNewBigramEntryToBigramList(const int bigramTargetPos, const int probability, bool addNewBigramEntryToBigramList(const int bigramTargetPos, const int probability,

View File

@ -20,12 +20,27 @@
#include "defines.h" #include "defines.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h"
#include "utils/hash_map_compat.h"
namespace latinime { namespace latinime {
// Interface class used to write PtNode information. // Interface class used to write PtNode information.
class PtNodeWriter { class PtNodeWriter {
public: public:
typedef hash_map_compat<int, int> PtNodeArrayPositionRelocationMap;
typedef hash_map_compat<int, int> PtNodePositionRelocationMap;
struct DictPositionRelocationMap {
public:
DictPositionRelocationMap()
: mPtNodeArrayPositionRelocationMap(), mPtNodePositionRelocationMap() {}
PtNodeArrayPositionRelocationMap mPtNodeArrayPositionRelocationMap;
PtNodePositionRelocationMap mPtNodePositionRelocationMap;
private:
DISALLOW_COPY_AND_ASSIGN(DictPositionRelocationMap);
};
virtual ~PtNodeWriter() {} virtual ~PtNodeWriter() {}
virtual bool markPtNodeAsDeleted(const PtNodeParams *const toBeUpdatedPtNodeParams) = 0; virtual bool markPtNodeAsDeleted(const PtNodeParams *const toBeUpdatedPtNodeParams) = 0;
@ -49,6 +64,13 @@ class PtNodeWriter {
virtual bool removeBigramEntry(const PtNodeParams *const sourcePtNodeParams, virtual bool removeBigramEntry(const PtNodeParams *const sourcePtNodeParams,
const PtNodeParams *const targetPtNodeParam) = 0; const PtNodeParams *const targetPtNodeParam) = 0;
virtual bool updateAllBigramEntriesAndDeleteUselessEntries(
const PtNodeParams *const sourcePtNodeParams, int *const outBigramEntryCount) = 0;
virtual bool updateAllPositionFields(const PtNodeParams *const toBeUpdatedPtNodeParams,
const DictPositionRelocationMap *const dictPositionRelocationMap,
int *const outBigramEntryCount) = 0;
protected: protected:
PtNodeWriter() {}; PtNodeWriter() {};

View File

@ -34,10 +34,8 @@ bool DynamicPatriciaTrieGcEventListeners
const int newProbability = const int newProbability =
ForgettingCurveUtils::getEncodedProbabilityToSave(ptNodeParams->getProbability(), ForgettingCurveUtils::getEncodedProbabilityToSave(ptNodeParams->getProbability(),
mHeaderPolicy); mHeaderPolicy);
int writingPos = ptNodeParams->getProbabilityFieldPos();
// Update probability. // Update probability.
if (!DynamicPatriciaTrieWritingUtils::writeProbabilityAndAdvancePosition( if (!mPtNodeWriter->updatePtNodeProbability(ptNodeParams, newProbability)) {
mBuffer, newProbability, &writingPos)) {
return false; return false;
} }
if (!ForgettingCurveUtils::isValidEncodedProbability(newProbability)) { if (!ForgettingCurveUtils::isValidEncodedProbability(newProbability)) {
@ -48,9 +46,8 @@ bool DynamicPatriciaTrieGcEventListeners
isUselessPtNode = false; isUselessPtNode = false;
} else if (ptNodeParams->isTerminal()) { } else if (ptNodeParams->isTerminal()) {
// Remove children as all children are useless. // Remove children as all children are useless.
int writingPos = ptNodeParams->getChildrenPosFieldPos(); if (!mPtNodeWriter->updateChildrenPosition(ptNodeParams,
if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition( NOT_A_DICT_POS /* newChildrenPosition */)) {
mBuffer, NOT_A_DICT_POS /* childrenPosition */, &writingPos)) {
return false; return false;
} }
} }
@ -74,7 +71,7 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateBigramProbabilit
int pos = ptNodeParams->getBigramsPos(); int pos = ptNodeParams->getBigramsPos();
if (pos != NOT_A_DICT_POS) { if (pos != NOT_A_DICT_POS) {
int bigramEntryCount = 0; int bigramEntryCount = 0;
if (!mBigramPolicy->updateAllBigramEntriesAndDeleteUselessEntries(&pos, if (!mPtNodeWriter->updateAllBigramEntriesAndDeleteUselessEntries(ptNodeParams,
&bigramEntryCount)) { &bigramEntryCount)) {
return false; return false;
} }
@ -90,8 +87,7 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNo
mValidPtNodeCount = 0; mValidPtNodeCount = 0;
int writingPos = mBufferToWrite->getTailPosition(); int writingPos = mBufferToWrite->getTailPosition();
mDictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.insert( mDictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.insert(
DynamicPatriciaTrieWritingHelper::PtNodeArrayPositionRelocationMap::value_type( PtNodeWriter::PtNodeArrayPositionRelocationMap::value_type(ptNodeArrayPos, writingPos));
ptNodeArrayPos, writingPos));
// Writes dummy PtNode array size because arrays can have a forward link or needles PtNodes. // Writes dummy PtNode array size because arrays can have a forward link or needles PtNodes.
// This field will be updated later in onReadingPtNodeArrayTail() with actual PtNode count. // This field will be updated later in onReadingPtNodeArrayTail() with actual PtNode count.
mPtNodeArraySizeFieldPos = writingPos; mPtNodeArraySizeFieldPos = writingPos;
@ -122,13 +118,13 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNo
if (ptNodeParams->isDeleted()) { if (ptNodeParams->isDeleted()) {
// Current PtNode is not written in new buffer because it has been deleted. // Current PtNode is not written in new buffer because it has been deleted.
mDictPositionRelocationMap->mPtNodePositionRelocationMap.insert( mDictPositionRelocationMap->mPtNodePositionRelocationMap.insert(
DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap::value_type( PtNodeWriter::PtNodePositionRelocationMap::value_type(
ptNodeParams->getHeadPos(), NOT_A_DICT_POS)); ptNodeParams->getHeadPos(), NOT_A_DICT_POS));
return true; return true;
} }
int writingPos = mBufferToWrite->getTailPosition(); int writingPos = mBufferToWrite->getTailPosition();
mDictPositionRelocationMap->mPtNodePositionRelocationMap.insert( mDictPositionRelocationMap->mPtNodePositionRelocationMap.insert(
DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap::value_type( PtNodeWriter::PtNodePositionRelocationMap::value_type(
ptNodeParams->getHeadPos(), writingPos)); ptNodeParams->getHeadPos(), writingPos));
mValidPtNodeCount++; mValidPtNodeCount++;
// Writes current PtNode. // Writes current PtNode.
@ -138,51 +134,15 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNo
bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateAllPositionFields bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateAllPositionFields
::onVisitingPtNode(const PtNodeParams *const ptNodeParams) { ::onVisitingPtNode(const PtNodeParams *const ptNodeParams) {
// Updates parent position. // Updates parent position.
int parentPos = ptNodeParams->getParentPos(); int bigramCount = 0;
if (parentPos != NOT_A_DICT_POS) { if (!mPtNodeWriter->updateAllPositionFields(ptNodeParams, mDictPositionRelocationMap,
DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap::const_iterator it = &bigramCount)) {
mDictPositionRelocationMap->mPtNodePositionRelocationMap.find(parentPos);
if (it != mDictPositionRelocationMap->mPtNodePositionRelocationMap.end()) {
parentPos = it->second;
}
}
int writingPos = ptNodeParams->getHeadPos()
+ DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE;
// Write updated parent offset.
if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition(mBufferToWrite,
parentPos, ptNodeParams->getHeadPos(), &writingPos)) {
return false; return false;
} }
mBigramCount += bigramCount;
// Updates children position.
int childrenPos = ptNodeParams->getChildrenPos();
if (childrenPos != NOT_A_DICT_POS) {
DynamicPatriciaTrieWritingHelper::PtNodeArrayPositionRelocationMap::const_iterator it =
mDictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.find(childrenPos);
if (it != mDictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.end()) {
childrenPos = it->second;
}
}
writingPos = ptNodeParams->getChildrenPosFieldPos();
if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(mBufferToWrite,
childrenPos, &writingPos)) {
return false;
}
// Updates bigram target PtNode positions in the bigram list.
int bigramsPos = ptNodeParams->getBigramsPos();
if (bigramsPos != NOT_A_DICT_POS) {
int bigramEntryCount;
if (!mBigramPolicy->updateAllBigramTargetPtNodePositions(&bigramsPos,
&mDictPositionRelocationMap->mPtNodePositionRelocationMap, &bigramEntryCount)) {
return false;
}
mBigramCount += bigramEntryCount;
}
if (ptNodeParams->isTerminal()) { if (ptNodeParams->isTerminal()) {
mUnigramCount++; mUnigramCount++;
} }
return true; return true;
} }

View File

@ -20,9 +20,8 @@
#include <vector> #include <vector>
#include "defines.h" #include "defines.h"
#include "suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h" #include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_helper.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "utils/hash_map_compat.h" #include "utils/hash_map_compat.h"
@ -32,6 +31,7 @@ class DictionaryHeaderStructurePolicy;
class PtNodeWriter; class PtNodeWriter;
class PtNodeParams; class PtNodeParams;
// TODO: Move to pt_common.
class DynamicPatriciaTrieGcEventListeners { class DynamicPatriciaTrieGcEventListeners {
public: public:
// Updates all PtNodes that can be reached from the root. Checks if each PtNode is useless or // Updates all PtNodes that can be reached from the root. Checks if each PtNode is useless or
@ -91,9 +91,8 @@ class DynamicPatriciaTrieGcEventListeners {
class TraversePolicyToUpdateBigramProbability class TraversePolicyToUpdateBigramProbability
: public DynamicPatriciaTrieReadingHelper::TraversingEventListener { : public DynamicPatriciaTrieReadingHelper::TraversingEventListener {
public: public:
TraversePolicyToUpdateBigramProbability( TraversePolicyToUpdateBigramProbability(PtNodeWriter *const ptNodeWriter)
DynamicBigramListPolicy *const bigramPolicy) : mPtNodeWriter(ptNodeWriter), mValidBigramEntryCount(0) {}
: mBigramPolicy(bigramPolicy), mValidBigramEntryCount(0) {}
bool onAscend() { return true; } bool onAscend() { return true; }
@ -110,7 +109,7 @@ class DynamicPatriciaTrieGcEventListeners {
private: private:
DISALLOW_IMPLICIT_CONSTRUCTORS(TraversePolicyToUpdateBigramProbability); DISALLOW_IMPLICIT_CONSTRUCTORS(TraversePolicyToUpdateBigramProbability);
DynamicBigramListPolicy *const mBigramPolicy; PtNodeWriter *const mPtNodeWriter;
int mValidBigramEntryCount; int mValidBigramEntryCount;
}; };
@ -119,8 +118,7 @@ class DynamicPatriciaTrieGcEventListeners {
public: public:
TraversePolicyToPlaceAndWriteValidPtNodesToBuffer( TraversePolicyToPlaceAndWriteValidPtNodesToBuffer(
PtNodeWriter *const ptNodeWriter, BufferWithExtendableBuffer *const bufferToWrite, PtNodeWriter *const ptNodeWriter, BufferWithExtendableBuffer *const bufferToWrite,
DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const PtNodeWriter::DictPositionRelocationMap *const dictPositionRelocationMap)
dictPositionRelocationMap)
: mPtNodeWriter(ptNodeWriter), mBufferToWrite(bufferToWrite), : mPtNodeWriter(ptNodeWriter), mBufferToWrite(bufferToWrite),
mDictPositionRelocationMap(dictPositionRelocationMap), mValidPtNodeCount(0), mDictPositionRelocationMap(dictPositionRelocationMap), mValidPtNodeCount(0),
mPtNodeArraySizeFieldPos(NOT_A_DICT_POS) {}; mPtNodeArraySizeFieldPos(NOT_A_DICT_POS) {};
@ -138,8 +136,7 @@ class DynamicPatriciaTrieGcEventListeners {
PtNodeWriter *const mPtNodeWriter; PtNodeWriter *const mPtNodeWriter;
BufferWithExtendableBuffer *const mBufferToWrite; BufferWithExtendableBuffer *const mBufferToWrite;
DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const PtNodeWriter::DictPositionRelocationMap *const mDictPositionRelocationMap;
mDictPositionRelocationMap;
int mValidPtNodeCount; int mValidPtNodeCount;
int mPtNodeArraySizeFieldPos; int mPtNodeArraySizeFieldPos;
}; };
@ -147,12 +144,9 @@ class DynamicPatriciaTrieGcEventListeners {
class TraversePolicyToUpdateAllPositionFields class TraversePolicyToUpdateAllPositionFields
: public DynamicPatriciaTrieReadingHelper::TraversingEventListener { : public DynamicPatriciaTrieReadingHelper::TraversingEventListener {
public: public:
TraversePolicyToUpdateAllPositionFields( TraversePolicyToUpdateAllPositionFields(PtNodeWriter *const ptNodeWriter,
DynamicBigramListPolicy *const bigramPolicy, const PtNodeWriter::DictPositionRelocationMap *const dictPositionRelocationMap)
BufferWithExtendableBuffer *const bufferToWrite, : mPtNodeWriter(ptNodeWriter),
const DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const
dictPositionRelocationMap)
: mBigramPolicy(bigramPolicy), mBufferToWrite(bufferToWrite),
mDictPositionRelocationMap(dictPositionRelocationMap), mUnigramCount(0), mDictPositionRelocationMap(dictPositionRelocationMap), mUnigramCount(0),
mBigramCount(0) {}; mBigramCount(0) {};
@ -175,10 +169,8 @@ class DynamicPatriciaTrieGcEventListeners {
private: private:
DISALLOW_IMPLICIT_CONSTRUCTORS(TraversePolicyToUpdateAllPositionFields); DISALLOW_IMPLICIT_CONSTRUCTORS(TraversePolicyToUpdateAllPositionFields);
DynamicBigramListPolicy *const mBigramPolicy; PtNodeWriter *const mPtNodeWriter;
BufferWithExtendableBuffer *const mBufferToWrite; const PtNodeWriter::DictPositionRelocationMap *const mDictPositionRelocationMap;
const DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const
mDictPositionRelocationMap;
int mUnigramCount; int mUnigramCount;
int mBigramCount; int mBigramCount;
}; };

View File

@ -233,4 +233,60 @@ bool DynamicPatriciaTrieNodeWriter::removeBigramEntry(
targetPtNodeParam->getHeadPos()); targetPtNodeParam->getHeadPos());
} }
bool DynamicPatriciaTrieNodeWriter::updateAllBigramEntriesAndDeleteUselessEntries(
const PtNodeParams *const sourcePtNodeParams, int *const outBigramEntryCount) {
int bigramListPos = sourcePtNodeParams->getBigramsPos();
return mBigramPolicy->updateAllBigramEntriesAndDeleteUselessEntries(&bigramListPos,
outBigramEntryCount);
}
bool DynamicPatriciaTrieNodeWriter::updateAllPositionFields(
const PtNodeParams *const toBeUpdatedPtNodeParams,
const DictPositionRelocationMap *const dictPositionRelocationMap,
int *const outBigramEntryCount) {
int parentPos = toBeUpdatedPtNodeParams->getParentPos();
if (parentPos != NOT_A_DICT_POS) {
PtNodeWriter::PtNodePositionRelocationMap::const_iterator it =
dictPositionRelocationMap->mPtNodePositionRelocationMap.find(parentPos);
if (it != dictPositionRelocationMap->mPtNodePositionRelocationMap.end()) {
parentPos = it->second;
}
}
int writingPos = toBeUpdatedPtNodeParams->getHeadPos()
+ DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE;
// Write an updated offset to the parent PtNode.
if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition(mBuffer,
parentPos, toBeUpdatedPtNodeParams->getHeadPos(), &writingPos)) {
return false;
}
// Updates children position that is a PtNodeArray position.
int childrenPos = toBeUpdatedPtNodeParams->getChildrenPos();
if (childrenPos != NOT_A_DICT_POS) {
PtNodeWriter::PtNodeArrayPositionRelocationMap::const_iterator it =
dictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.find(childrenPos);
if (it != dictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.end()) {
childrenPos = it->second;
}
}
if (!updateChildrenPosition(toBeUpdatedPtNodeParams, childrenPos)) {
return false;
}
// Updates bigram target PtNode positions in the bigram list.
int bigramsPos = toBeUpdatedPtNodeParams->getBigramsPos();
int bigramCount = 0;
if (bigramsPos != NOT_A_DICT_POS) {
int bigramEntryCount;
if (!mBigramPolicy->updateAllBigramTargetPtNodePositions(&bigramsPos,
&dictPositionRelocationMap->mPtNodePositionRelocationMap, &bigramEntryCount)) {
return false;
}
bigramCount += bigramEntryCount;
}
if (outBigramEntryCount) {
*outBigramEntryCount = bigramCount;
}
return true;
}
} }

View File

@ -66,6 +66,13 @@ class DynamicPatriciaTrieNodeWriter : public PtNodeWriter {
virtual bool removeBigramEntry(const PtNodeParams *const sourcePtNodeParams, virtual bool removeBigramEntry(const PtNodeParams *const sourcePtNodeParams,
const PtNodeParams *const targetPtNodeParam); const PtNodeParams *const targetPtNodeParam);
virtual bool updateAllBigramEntriesAndDeleteUselessEntries(
const PtNodeParams *const sourcePtNodeParams, int *const outBigramEntryCount);
virtual bool updateAllPositionFields(const PtNodeParams *const toBeUpdatedPtNodeParams,
const DictPositionRelocationMap *const dictPositionRelocationMap,
int *const outBigramEntryCount);
private: private:
DISALLOW_COPY_AND_ASSIGN(DynamicPatriciaTrieNodeWriter); DISALLOW_COPY_AND_ASSIGN(DynamicPatriciaTrieNodeWriter);

View File

@ -92,7 +92,7 @@ bool DynamicPatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos);
DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateBigramProbability DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateBigramProbability
traversePolicyToUpdateBigramProbability(mBigramPolicy); traversePolicyToUpdateBigramProbability(mPtNodeWriter);
if (!readingHelper.traverseAllPtNodesInPostorderDepthFirstManner( if (!readingHelper.traverseAllPtNodesInPostorderDepthFirstManner(
&traversePolicyToUpdateBigramProbability)) { &traversePolicyToUpdateBigramProbability)) {
return false; return false;
@ -103,10 +103,10 @@ bool DynamicPatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
} }
// Mapping from positions in mBuffer to positions in bufferToWrite. // Mapping from positions in mBuffer to positions in bufferToWrite.
DictPositionRelocationMap dictPositionRelocationMap; PtNodeWriter::DictPositionRelocationMap dictPositionRelocationMap;
readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos);
DynamicPatriciaTrieNodeWriter newPtNodeWriter(bufferToWrite, DynamicPatriciaTrieNodeWriter newPtNodeWriter(bufferToWrite, &ptNodeReader, mBigramPolicy,
&ptNodeReader, mBigramPolicy, mShortcutPolicy); mShortcutPolicy);
DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer
traversePolicyToPlaceAndWriteValidPtNodesToBuffer(&newPtNodeWriter, bufferToWrite, traversePolicyToPlaceAndWriteValidPtNodesToBuffer(&newPtNodeWriter, bufferToWrite,
&dictPositionRelocationMap); &dictPositionRelocationMap);
@ -123,10 +123,11 @@ bool DynamicPatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
DynamicPatriciaTrieNodeReader newDictNodeReader(bufferToWrite, &newDictBigramPolicy, DynamicPatriciaTrieNodeReader newDictNodeReader(bufferToWrite, &newDictBigramPolicy,
&newDictShortcutPolicy); &newDictShortcutPolicy);
DynamicPatriciaTrieReadingHelper newDictReadingHelper(bufferToWrite, &newDictNodeReader); DynamicPatriciaTrieReadingHelper newDictReadingHelper(bufferToWrite, &newDictNodeReader);
DynamicPatriciaTrieNodeWriter newDictNodeWriter(bufferToWrite, &newDictNodeReader,
&newDictBigramPolicy, &newDictShortcutPolicy);
newDictReadingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); newDictReadingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos);
DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateAllPositionFields DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateAllPositionFields
traversePolicyToUpdateAllPositionFields(&newDictBigramPolicy, bufferToWrite, traversePolicyToUpdateAllPositionFields(&newDictNodeWriter, &dictPositionRelocationMap);
&dictPositionRelocationMap);
if (!newDictReadingHelper.traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner( if (!newDictReadingHelper.traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner(
&traversePolicyToUpdateAllPositionFields)) { &traversePolicyToUpdateAllPositionFields)) {
return false; return false;

View File

@ -20,7 +20,6 @@
#include <stdint.h> #include <stdint.h>
#include "defines.h" #include "defines.h"
#include "utils/hash_map_compat.h"
namespace latinime { namespace latinime {
@ -35,20 +34,6 @@ class PtNodeWriter;
// TODO: Make it independent from a particular format and move to pt_common. // TODO: Make it independent from a particular format and move to pt_common.
class DynamicPatriciaTrieWritingHelper { class DynamicPatriciaTrieWritingHelper {
public: public:
typedef hash_map_compat<int, int> PtNodeArrayPositionRelocationMap;
typedef hash_map_compat<int, int> PtNodePositionRelocationMap;
struct DictPositionRelocationMap {
public:
DictPositionRelocationMap()
: mPtNodeArrayPositionRelocationMap(), mPtNodePositionRelocationMap() {}
PtNodeArrayPositionRelocationMap mPtNodeArrayPositionRelocationMap;
PtNodePositionRelocationMap mPtNodePositionRelocationMap;
private:
DISALLOW_COPY_AND_ASSIGN(DictPositionRelocationMap);
};
static const size_t MAX_DICTIONARY_SIZE; static const size_t MAX_DICTIONARY_SIZE;
DynamicPatriciaTrieWritingHelper(BufferWithExtendableBuffer *const buffer, DynamicPatriciaTrieWritingHelper(BufferWithExtendableBuffer *const buffer,

View File

@ -196,4 +196,18 @@ bool Ver4PatriciaTrieNodeWriter::removeBigramEntry(
targetPtNodeParam->getTerminalId()); targetPtNodeParam->getTerminalId());
} }
bool Ver4PatriciaTrieNodeWriter::updateAllBigramEntriesAndDeleteUselessEntries(
const PtNodeParams *const sourcePtNodeParams, int *const outBigramEntryCount) {
// TODO: Implement.
return false;
}
bool Ver4PatriciaTrieNodeWriter::updateAllPositionFields(
const PtNodeParams *const toBeUpdatedPtNodeParams,
const DictPositionRelocationMap *const dictPositionRelocationMap,
int *const outBigramEntryCount) {
// TODO: Implement.
return false;
}
} }

View File

@ -67,6 +67,13 @@ class Ver4PatriciaTrieNodeWriter : public PtNodeWriter {
virtual bool removeBigramEntry(const PtNodeParams *const sourcePtNodeParams, virtual bool removeBigramEntry(const PtNodeParams *const sourcePtNodeParams,
const PtNodeParams *const targetPtNodeParam); const PtNodeParams *const targetPtNodeParam);
virtual bool updateAllBigramEntriesAndDeleteUselessEntries(
const PtNodeParams *const sourcePtNodeParams, int *const outBigramEntryCount);
virtual bool updateAllPositionFields(const PtNodeParams *const toBeUpdatedPtNodeParams,
const DictPositionRelocationMap *const dictPositionRelocationMap,
int *const outBigramEntryCount);
private: private:
DISALLOW_COPY_AND_ASSIGN(Ver4PatriciaTrieNodeWriter); DISALLOW_COPY_AND_ASSIGN(Ver4PatriciaTrieNodeWriter);
@ -78,7 +85,6 @@ class Ver4PatriciaTrieNodeWriter : public PtNodeWriter {
DynamicPatriciaTrieReadingHelper mReadingHelper; DynamicPatriciaTrieReadingHelper mReadingHelper;
Ver4BigramListPolicy *const mBigramPolicy; Ver4BigramListPolicy *const mBigramPolicy;
Ver4ShortcutListPolicy *const mShortcutPolicy; Ver4ShortcutListPolicy *const mShortcutPolicy;
}; };
} // namespace latinime } // namespace latinime
#endif /* LATINIME_VER4_PATRICIA_TRIE_NODE_WRITER_H */ #endif /* LATINIME_VER4_PATRICIA_TRIE_NODE_WRITER_H */

View File

@ -18,11 +18,16 @@
#include <cstring> #include <cstring>
#include "suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/header/header_policy.h" #include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/shortcut/ver4_shortcut_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/file_utils.h" #include "suggest/policyimpl/dictionary/utils/file_utils.h"
#include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
namespace latinime { namespace latinime {
@ -47,4 +52,47 @@ void Ver4PatriciaTrieWritingHelper::writeToDictFile(const char *const trieFilePa
mBuffers->flushHeaderAndDictBuffers(dirPath, &headerBuffer); mBuffers->flushHeaderAndDictBuffers(dirPath, &headerBuffer);
} }
void Ver4PatriciaTrieWritingHelper::writeToDictFileWithGC(const int rootPtNodeArrayPos,
const char *const trieFilePath, const HeaderPolicy *const headerPolicy,
const bool needsToDecay) {
Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers(Ver4DictBuffers::createVer4DictBuffers());
int unigramCount = 0;
int bigramCount = 0;
if (needsToDecay) {
ForgettingCurveUtils::sTimeKeeper.setCurrentTime();
}
if (!runGC(rootPtNodeArrayPos, headerPolicy, dictBuffers.get(), &unigramCount, &bigramCount,
needsToDecay)) {
return;
}
BufferWithExtendableBuffer headerBuffer(
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE);
if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, true /* updatesLastUpdatedTime */,
needsToDecay, unigramCount, bigramCount, 0 /* extendedRegionSize */)) {
return;
}
const int dirPathBufSize = strlen(trieFilePath) + 1 /* terminator */;
char dirPath[dirPathBufSize];
FileUtils::getDirPath(trieFilePath, dirPathBufSize, dirPath);
dictBuffers.get()->flushHeaderAndDictBuffers(dirPath, &headerBuffer);
}
bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
const HeaderPolicy *const headerPolicy, Ver4DictBuffers *const buffersToWrite,
int *const outUnigramCount, int *const outBigramCount, const bool needsToDecay) {
Ver4PatriciaTrieNodeReader ptNodeReader(mBuffers->getTrieBuffer(),
mBuffers->getProbabilityDictContent());
Ver4BigramListPolicy bigramPolicy(mBuffers->getUpdatableBigramDictContent(),
mBuffers->getTerminalPositionLookupTable());
Ver4ShortcutListPolicy shortcutPolicy(mBuffers->getShortcutDictContent(),
mBuffers->getTerminalPositionLookupTable());
Ver4PatriciaTrieNodeWriter ptNodeWriter(mBuffers->getWritableTrieBuffer(),
mBuffers, &ptNodeReader, &bigramPolicy, &shortcutPolicy);
DynamicPatriciaTrieReadingHelper readingHelper(mBuffers->getTrieBuffer(), &ptNodeReader);
readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos);
return true;
}
} // namespace latinime } // namespace latinime

View File

@ -32,9 +32,17 @@ class Ver4PatriciaTrieWritingHelper {
void writeToDictFile(const char *const trieFilePath, const HeaderPolicy *const headerPolicy, void writeToDictFile(const char *const trieFilePath, const HeaderPolicy *const headerPolicy,
const int unigramCount, const int bigramCount) const; const int unigramCount, const int bigramCount) const;
void writeToDictFileWithGC(const int rootPtNodeArrayPos,
const char *const trieFilePath, const HeaderPolicy *const headerPolicy,
const bool needsToDecay);
private: private:
DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4PatriciaTrieWritingHelper); DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4PatriciaTrieWritingHelper);
bool runGC(const int rootPtNodeArrayPos, const HeaderPolicy *const headerPolicy,
Ver4DictBuffers *const buffersToWrite, int *const outUnigramCount,
int *const outBigramCount, const bool needsToDecay);
Ver4DictBuffers *const mBuffers; Ver4DictBuffers *const mBuffers;
}; };
} // namespace latinime } // namespace latinime