am 5e797bd7: Merge "Implement update node probability method."
* commit '5e797bd7e29861bf60087b27634dc249bf0c70bc': Implement update node probability method.main
commit
ecdde8ee85
|
@ -43,8 +43,13 @@ void DynamicPatriciaTrieNodeReader::fetchNodeInfoFromBufferAndProcessMovedNode(c
|
|||
dictBuf, mFlags, MAX_WORD_LENGTH, &pos);
|
||||
}
|
||||
if (isTerminal()) {
|
||||
mProbabilityFieldPos = pos;
|
||||
if (usesAdditionalBuffer) {
|
||||
mProbabilityFieldPos += mBuffer->getOriginalBufferSize();
|
||||
}
|
||||
mProbability = PatriciaTrieReadingUtils::readProbabilityAndAdvancePosition(dictBuf, &pos);
|
||||
} else {
|
||||
mProbabilityFieldPos = NOT_A_DICT_POS;
|
||||
mProbability = NOT_A_PROBABILITY;
|
||||
}
|
||||
mChildrenPos = DynamicPatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition(
|
||||
|
|
|
@ -40,7 +40,8 @@ class DynamicPatriciaTrieNodeReader {
|
|||
const DictionaryShortcutsStructurePolicy *const shortcutsPolicy)
|
||||
: mBuffer(buffer), mBigramsPolicy(bigramsPolicy),
|
||||
mShortcutsPolicy(shortcutsPolicy), mNodePos(NOT_A_VALID_WORD_POS), mFlags(0),
|
||||
mParentPos(NOT_A_DICT_POS), mCodePointCount(0), mProbability(NOT_A_PROBABILITY),
|
||||
mParentPos(NOT_A_DICT_POS), mCodePointCount(0),
|
||||
mProbabilityFieldPos(NOT_A_DICT_POS), mProbability(NOT_A_PROBABILITY),
|
||||
mChildrenPos(NOT_A_DICT_POS), mShortcutPos(NOT_A_DICT_POS),
|
||||
mBigramPos(NOT_A_DICT_POS), mSiblingPos(NOT_A_VALID_WORD_POS) {}
|
||||
|
||||
|
@ -95,6 +96,10 @@ class DynamicPatriciaTrieNodeReader {
|
|||
}
|
||||
|
||||
// Probability
|
||||
AK_FORCE_INLINE int getProbabilityFieldPos() const {
|
||||
return mProbabilityFieldPos;
|
||||
}
|
||||
|
||||
AK_FORCE_INLINE int getProbability() const {
|
||||
return mProbability;
|
||||
}
|
||||
|
@ -129,6 +134,7 @@ class DynamicPatriciaTrieNodeReader {
|
|||
DynamicPatriciaTrieReadingUtils::NodeFlags mFlags;
|
||||
int mParentPos;
|
||||
uint8_t mCodePointCount;
|
||||
int mProbabilityFieldPos;
|
||||
int mProbability;
|
||||
int mChildrenPos;
|
||||
int mShortcutPos;
|
||||
|
|
|
@ -26,6 +26,9 @@
|
|||
|
||||
namespace latinime {
|
||||
|
||||
// TODO: Enable dynamic update and remove this flag.
|
||||
const bool DynamicPatriciaTrieWritingHelper::ENABLE_DYNAMIC_UPDATE = false;
|
||||
|
||||
bool DynamicPatriciaTrieWritingHelper::addUnigramWord(
|
||||
DynamicPatriciaTrieReadingHelper *const readingHelper,
|
||||
const int *const wordCodePoints, const int codePointCount, const int probability) {
|
||||
|
@ -56,13 +59,13 @@ bool DynamicPatriciaTrieWritingHelper::addUnigramWord(
|
|||
}
|
||||
// All characters are matched.
|
||||
if (codePointCount == readingHelper->getTotalCodePointCount()) {
|
||||
if (nodeReader->isTerminal()) {
|
||||
// TODO: Update probability.
|
||||
if (ENABLE_DYNAMIC_UPDATE) {
|
||||
setPtNodeProbability(nodeReader, probability,
|
||||
readingHelper->getMergedNodeCodePoints());
|
||||
} else {
|
||||
// TODO: Make it terminal and update probability.
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (!nodeReader->hasChildren()) {
|
||||
// TODO: Create children node array and add new node as a child.
|
||||
return false;
|
||||
|
@ -76,12 +79,14 @@ bool DynamicPatriciaTrieWritingHelper::addUnigramWord(
|
|||
return false;
|
||||
}
|
||||
int pos = readingHelper->getPosOfLastForwardLinkField();
|
||||
// TODO: Remove.
|
||||
return false;
|
||||
if (ENABLE_DYNAMIC_UPDATE) {
|
||||
return createAndInsertNodeIntoPtNodeArray(parentPos,
|
||||
wordCodePoints + readingHelper->getPrevTotalCodePointCount(),
|
||||
codePointCount - readingHelper->getPrevTotalCodePointCount(),
|
||||
probability, &pos);
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
bool DynamicPatriciaTrieWritingHelper::addBigramWords(const int word0Pos, const int word1Pos,
|
||||
|
@ -214,4 +219,30 @@ bool DynamicPatriciaTrieWritingHelper::createAndInsertNodeIntoPtNodeArray(const
|
|||
return true;
|
||||
}
|
||||
|
||||
bool DynamicPatriciaTrieWritingHelper::setPtNodeProbability(
|
||||
const DynamicPatriciaTrieNodeReader *const originalPtNode, const int probability,
|
||||
const int *const codePoints) {
|
||||
if (originalPtNode->isTerminal()) {
|
||||
// Overwrites the probability.
|
||||
int probabilityFieldPos = originalPtNode->getProbabilityFieldPos();
|
||||
if (!DynamicPatriciaTrieWritingUtils::writeProbabilityAndAdvancePosition(mBuffer,
|
||||
probability, &probabilityFieldPos)) {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
// Make the node terminal and write the probability.
|
||||
int movedPos = mBuffer->getTailPosition();
|
||||
if (!markNodeAsMovedAndSetPosition(originalPtNode, movedPos)) {
|
||||
return false;
|
||||
}
|
||||
if (!writeNodeToBuffer(originalPtNode->isBlacklisted(), originalPtNode->isNotAWord(),
|
||||
originalPtNode->getParentPos(), codePoints, originalPtNode->getCodePointCount(),
|
||||
probability, originalPtNode->getChildrenPos(), originalPtNode->getBigramsPos(),
|
||||
originalPtNode->getShortcutPos(), &movedPos)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace latinime
|
||||
|
|
|
@ -49,6 +49,7 @@ class DynamicPatriciaTrieWritingHelper {
|
|||
private:
|
||||
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieWritingHelper);
|
||||
|
||||
static const bool ENABLE_DYNAMIC_UPDATE;
|
||||
BufferWithExtendableBuffer *const mBuffer;
|
||||
DynamicBigramListPolicy *const mBigramPolicy;
|
||||
DynamicShortcutListPolicy *const mShortcutPolicy;
|
||||
|
@ -63,6 +64,9 @@ class DynamicPatriciaTrieWritingHelper {
|
|||
|
||||
bool createAndInsertNodeIntoPtNodeArray(const int parentPos, const int *const nodeCodePoints,
|
||||
const int nodeCodePointCount, const int probability, int *const forwardLinkFieldPos);
|
||||
|
||||
bool setPtNodeProbability(const DynamicPatriciaTrieNodeReader *const originalNode,
|
||||
const int probability, const int *const codePoints);
|
||||
};
|
||||
} // namespace latinime
|
||||
#endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H */
|
||||
|
|
Loading…
Reference in New Issue