am fd10db04
: Move shortcut iteration methods to poilcy.
* commit 'fd10db04e02ddad88d0c6fca82583493955a7c7e': Move shortcut iteration methods to poilcy.
This commit is contained in:
commit
ad49fa9f2c
13 changed files with 203 additions and 90 deletions
|
@ -20,7 +20,6 @@
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
|
|
||||||
#include "defines.h"
|
#include "defines.h"
|
||||||
#include "suggest/core/dictionary/binary_dictionary_info.h"
|
|
||||||
#include "suggest/core/dictionary/byte_array_utils.h"
|
#include "suggest/core/dictionary/byte_array_utils.h"
|
||||||
|
|
||||||
namespace latinime {
|
namespace latinime {
|
||||||
|
@ -47,16 +46,14 @@ class BinaryDictionaryTerminalAttributesReadingUtils {
|
||||||
// This method returns the size of the shortcut list region excluding the shortcut list size
|
// This method returns the size of the shortcut list region excluding the shortcut list size
|
||||||
// field at the beginning.
|
// field at the beginning.
|
||||||
static AK_FORCE_INLINE int getShortcutListSizeAndForwardPointer(
|
static AK_FORCE_INLINE int getShortcutListSizeAndForwardPointer(
|
||||||
const BinaryDictionaryInfo *const binaryDictionaryInfo, int *const pos) {
|
const uint8_t *const dictRoot, int *const pos) {
|
||||||
// readUint16andAdvancePosition() returns an offset *including* the uint16 field itself.
|
// readUint16andAdvancePosition() returns an offset *including* the uint16 field itself.
|
||||||
return ByteArrayUtils::readUint16AndAdvancePosition(
|
return ByteArrayUtils::readUint16AndAdvancePosition(dictRoot, pos)
|
||||||
binaryDictionaryInfo->getDictRoot(), pos) - SHORTCUT_LIST_SIZE_FIELD_SIZE;
|
- SHORTCUT_LIST_SIZE_FIELD_SIZE;
|
||||||
}
|
}
|
||||||
|
|
||||||
static AK_FORCE_INLINE void skipShortcuts(
|
static AK_FORCE_INLINE void skipShortcuts(const uint8_t *const dictRoot, int *const pos) {
|
||||||
const BinaryDictionaryInfo *const binaryDictionaryInfo, int *const pos) {
|
const int shortcutListSize = getShortcutListSizeAndForwardPointer(dictRoot, pos);
|
||||||
const int shortcutListSize = getShortcutListSizeAndForwardPointer(
|
|
||||||
binaryDictionaryInfo, pos);
|
|
||||||
*pos += shortcutListSize;
|
*pos += shortcutListSize;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -65,10 +62,9 @@ class BinaryDictionaryTerminalAttributesReadingUtils {
|
||||||
}
|
}
|
||||||
|
|
||||||
static AK_FORCE_INLINE int readShortcutTarget(
|
static AK_FORCE_INLINE int readShortcutTarget(
|
||||||
const BinaryDictionaryInfo *const binaryDictionaryInfo, const int maxLength,
|
const uint8_t *const dictRoot, const int maxLength, int *const outWord,
|
||||||
int *const outWord, int *const pos) {
|
int *const pos) {
|
||||||
return ByteArrayUtils::readStringAndAdvancePosition(
|
return ByteArrayUtils::readStringAndAdvancePosition(dictRoot, maxLength, outWord, pos);
|
||||||
binaryDictionaryInfo->getDictRoot(), maxLength, outWord, pos);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
|
@ -19,8 +19,7 @@
|
||||||
|
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
|
|
||||||
#include "suggest/core/dictionary/binary_dictionary_info.h"
|
#include "suggest/core/policy/dictionary_shortcuts_structure_policy.h"
|
||||||
#include "suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.h"
|
|
||||||
|
|
||||||
namespace latinime {
|
namespace latinime {
|
||||||
|
|
||||||
|
@ -33,9 +32,9 @@ class TerminalAttributes {
|
||||||
public:
|
public:
|
||||||
class ShortcutIterator {
|
class ShortcutIterator {
|
||||||
public:
|
public:
|
||||||
ShortcutIterator(const BinaryDictionaryInfo *const binaryDictionaryInfo,
|
ShortcutIterator(const DictionaryShortcutsStructurePolicy *const shortcutStructurePolicy,
|
||||||
const int shortcutPos, const bool hasShortcutList)
|
const int shortcutPos, const bool hasShortcutList)
|
||||||
: mBinaryDictionaryInfo(binaryDictionaryInfo), mPos(shortcutPos),
|
: mShortcutStructurePolicy(shortcutStructurePolicy), mPos(shortcutPos),
|
||||||
mHasNextShortcutTarget(hasShortcutList) {}
|
mHasNextShortcutTarget(hasShortcutList) {}
|
||||||
|
|
||||||
inline bool hasNextShortcutTarget() const {
|
inline bool hasNextShortcutTarget() const {
|
||||||
|
@ -47,46 +46,34 @@ class TerminalAttributes {
|
||||||
AK_FORCE_INLINE void nextShortcutTarget(
|
AK_FORCE_INLINE void nextShortcutTarget(
|
||||||
const int maxDepth, int *const outTarget, int *const outTargetLength,
|
const int maxDepth, int *const outTarget, int *const outTargetLength,
|
||||||
bool *const outIsWhitelist) {
|
bool *const outIsWhitelist) {
|
||||||
const BinaryDictionaryTerminalAttributesReadingUtils::ShortcutFlags flags =
|
mShortcutStructurePolicy->getNextShortcut(maxDepth, outTarget, outTargetLength,
|
||||||
BinaryDictionaryTerminalAttributesReadingUtils::getFlagsAndForwardPointer(
|
outIsWhitelist, &mHasNextShortcutTarget, &mPos);
|
||||||
mBinaryDictionaryInfo->getDictRoot(), &mPos);
|
|
||||||
mHasNextShortcutTarget =
|
|
||||||
BinaryDictionaryTerminalAttributesReadingUtils::hasNext(flags);
|
|
||||||
if (outIsWhitelist) {
|
|
||||||
*outIsWhitelist =
|
|
||||||
BinaryDictionaryTerminalAttributesReadingUtils::isWhitelist(flags);
|
|
||||||
}
|
|
||||||
if (outTargetLength) {
|
|
||||||
*outTargetLength =
|
|
||||||
BinaryDictionaryTerminalAttributesReadingUtils::readShortcutTarget(
|
|
||||||
mBinaryDictionaryInfo, maxDepth, outTarget, &mPos);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
const BinaryDictionaryInfo *const mBinaryDictionaryInfo;
|
const DictionaryShortcutsStructurePolicy *const mShortcutStructurePolicy;
|
||||||
int mPos;
|
int mPos;
|
||||||
bool mHasNextShortcutTarget;
|
bool mHasNextShortcutTarget;
|
||||||
};
|
};
|
||||||
|
|
||||||
TerminalAttributes(const BinaryDictionaryInfo *const binaryDictionaryInfo,
|
TerminalAttributes(const DictionaryShortcutsStructurePolicy *const shortcutStructurePolicy,
|
||||||
const int shortcutPos)
|
const int shortcutPos)
|
||||||
: mBinaryDictionaryInfo(binaryDictionaryInfo), mShortcutListSizePos(shortcutPos) {}
|
: mShortcutStructurePolicy(shortcutStructurePolicy),
|
||||||
|
mShortcutListSizePos(shortcutPos) {}
|
||||||
|
|
||||||
inline ShortcutIterator getShortcutIterator() const {
|
inline ShortcutIterator getShortcutIterator() const {
|
||||||
int shortcutPos = mShortcutListSizePos;
|
int shortcutPos = mShortcutListSizePos;
|
||||||
const bool hasShortcutList = shortcutPos != NOT_A_DICT_POS;
|
const bool hasShortcutList = shortcutPos != NOT_A_DICT_POS;
|
||||||
if (hasShortcutList) {
|
if (hasShortcutList) {
|
||||||
BinaryDictionaryTerminalAttributesReadingUtils::getShortcutListSizeAndForwardPointer(
|
shortcutPos = mShortcutStructurePolicy->getStartPos(shortcutPos);
|
||||||
mBinaryDictionaryInfo, &shortcutPos);
|
|
||||||
}
|
}
|
||||||
// shortcutPos is never used if hasShortcutList is false.
|
// shortcutPos is never used if hasShortcutList is false.
|
||||||
return ShortcutIterator(mBinaryDictionaryInfo, shortcutPos, hasShortcutList);
|
return ShortcutIterator(mShortcutStructurePolicy, shortcutPos, hasShortcutList);
|
||||||
}
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
DISALLOW_IMPLICIT_CONSTRUCTORS(TerminalAttributes);
|
DISALLOW_IMPLICIT_CONSTRUCTORS(TerminalAttributes);
|
||||||
const BinaryDictionaryInfo *const mBinaryDictionaryInfo;
|
const DictionaryShortcutsStructurePolicy *const mShortcutStructurePolicy;
|
||||||
const int mShortcutListSizePos;
|
const int mShortcutListSizePos;
|
||||||
};
|
};
|
||||||
} // namespace latinime
|
} // namespace latinime
|
||||||
|
|
|
@ -0,0 +1,46 @@
|
||||||
|
/*
|
||||||
|
* Copyright (C) 2013, The Android Open Source Project
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#ifndef LATINIME_DICTIONARY_SHORTCUTS_STRUCTURE_POLICY_H
|
||||||
|
#define LATINIME_DICTIONARY_SHORTCUTS_STRUCTURE_POLICY_H
|
||||||
|
|
||||||
|
#include "defines.h"
|
||||||
|
|
||||||
|
namespace latinime {
|
||||||
|
|
||||||
|
/*
|
||||||
|
* This class abstracts structure of shortcuts.
|
||||||
|
*/
|
||||||
|
class DictionaryShortcutsStructurePolicy {
|
||||||
|
public:
|
||||||
|
virtual ~DictionaryShortcutsStructurePolicy() {}
|
||||||
|
|
||||||
|
virtual int getStartPos(const int pos) const = 0;
|
||||||
|
|
||||||
|
virtual void getNextShortcut(const int maxCodePointCount, int *const outCodePoint,
|
||||||
|
int *const outCodePointCount, bool *const outIsWhitelist, bool *const outHasNext,
|
||||||
|
int *const pos) const = 0;
|
||||||
|
|
||||||
|
virtual void skipAllShortcuts(int *const pos) const = 0;
|
||||||
|
|
||||||
|
protected:
|
||||||
|
DictionaryShortcutsStructurePolicy() {}
|
||||||
|
|
||||||
|
private:
|
||||||
|
DISALLOW_COPY_AND_ASSIGN(DictionaryShortcutsStructurePolicy);
|
||||||
|
};
|
||||||
|
} // namespace latinime
|
||||||
|
#endif /* LATINIME_DICTIONARY_SHORTCUTS_STRUCTURE_POLICY_H */
|
|
@ -24,6 +24,7 @@ namespace latinime {
|
||||||
class DicNode;
|
class DicNode;
|
||||||
class DicNodeVector;
|
class DicNodeVector;
|
||||||
class DictionaryBigramsStructurePolicy;
|
class DictionaryBigramsStructurePolicy;
|
||||||
|
class DictionaryShortcutsStructurePolicy;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* This class abstracts structure of dictionaries.
|
* This class abstracts structure of dictionaries.
|
||||||
|
@ -66,6 +67,8 @@ class DictionaryStructureWithBufferPolicy {
|
||||||
|
|
||||||
virtual const DictionaryBigramsStructurePolicy *getBigramsStructurePolicy() const = 0;
|
virtual const DictionaryBigramsStructurePolicy *getBigramsStructurePolicy() const = 0;
|
||||||
|
|
||||||
|
virtual const DictionaryShortcutsStructurePolicy *getShortcutsStructurePolicy() const = 0;
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
DictionaryStructureWithBufferPolicy() {}
|
DictionaryStructureWithBufferPolicy() {}
|
||||||
|
|
||||||
|
|
|
@ -19,6 +19,7 @@
|
||||||
#include "suggest/core/dicnode/dic_node.h"
|
#include "suggest/core/dicnode/dic_node.h"
|
||||||
#include "suggest/core/dicnode/dic_node_priority_queue.h"
|
#include "suggest/core/dicnode/dic_node_priority_queue.h"
|
||||||
#include "suggest/core/dicnode/dic_node_vector.h"
|
#include "suggest/core/dicnode/dic_node_vector.h"
|
||||||
|
// TODO: Use DictionaryStructurePolicy instead of BinaryDictionaryInfo.
|
||||||
#include "suggest/core/dictionary/binary_dictionary_info.h"
|
#include "suggest/core/dictionary/binary_dictionary_info.h"
|
||||||
#include "suggest/core/dictionary/dictionary.h"
|
#include "suggest/core/dictionary/dictionary.h"
|
||||||
#include "suggest/core/dictionary/digraph_utils.h"
|
#include "suggest/core/dictionary/digraph_utils.h"
|
||||||
|
@ -211,11 +212,11 @@ int Suggest::outputSuggestions(DicTraverseSession *traverseSession, int *frequen
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!terminalDicNode->hasMultipleWords()) {
|
if (!terminalDicNode->hasMultipleWords()) {
|
||||||
const BinaryDictionaryInfo *const binaryDictionaryInfo =
|
const DictionaryStructureWithBufferPolicy *const structurePolicy =
|
||||||
traverseSession->getBinaryDictionaryInfo();
|
traverseSession->getBinaryDictionaryInfo()->getStructurePolicy();
|
||||||
const TerminalAttributes terminalAttributes(traverseSession->getBinaryDictionaryInfo(),
|
const TerminalAttributes terminalAttributes(
|
||||||
binaryDictionaryInfo->getStructurePolicy()->getShortcutPositionOfNode(
|
structurePolicy->getShortcutsStructurePolicy(),
|
||||||
terminalDicNode->getPos()));
|
structurePolicy->getShortcutPositionOfNode(terminalDicNode->getPos()));
|
||||||
// Shortcut is not supported for multiple words suggestions.
|
// Shortcut is not supported for multiple words suggestions.
|
||||||
// TODO: Check shortcuts during traversal for multiple words suggestions.
|
// TODO: Check shortcuts during traversal for multiple words suggestions.
|
||||||
const bool sameAsTyped = TRAVERSAL->sameAsTyped(traverseSession, terminalDicNode);
|
const bool sameAsTyped = TRAVERSAL->sameAsTyped(traverseSession, terminalDicNode);
|
||||||
|
|
|
@ -28,11 +28,9 @@ namespace latinime {
|
||||||
const BinaryDictionaryInfo *const binaryDictionaryInfo) {
|
const BinaryDictionaryInfo *const binaryDictionaryInfo) {
|
||||||
switch (binaryDictionaryInfo->getFormat()) {
|
switch (binaryDictionaryInfo->getFormat()) {
|
||||||
case BinaryDictionaryFormatUtils::VERSION_2:
|
case BinaryDictionaryFormatUtils::VERSION_2:
|
||||||
return new PatriciaTriePolicy(binaryDictionaryInfo->getDictRoot(),
|
return new PatriciaTriePolicy(binaryDictionaryInfo->getDictRoot());
|
||||||
binaryDictionaryInfo);
|
|
||||||
case BinaryDictionaryFormatUtils::VERSION_3:
|
case BinaryDictionaryFormatUtils::VERSION_3:
|
||||||
return new DynamicPatriciaTriePolicy(binaryDictionaryInfo->getDictRoot(),
|
return new DynamicPatriciaTriePolicy(binaryDictionaryInfo->getDictRoot());
|
||||||
binaryDictionaryInfo);
|
|
||||||
default:
|
default:
|
||||||
ASSERT(false);
|
ASSERT(false);
|
||||||
return 0;
|
return 0;
|
||||||
|
|
|
@ -16,42 +16,40 @@
|
||||||
|
|
||||||
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h"
|
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h"
|
||||||
|
|
||||||
#include "suggest/core/dictionary/binary_dictionary_info.h"
|
|
||||||
#include "suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.h"
|
|
||||||
#include "suggest/core/policy/dictionary_bigrams_structure_policy.h"
|
#include "suggest/core/policy/dictionary_bigrams_structure_policy.h"
|
||||||
|
#include "suggest/core/policy/dictionary_shortcuts_structure_policy.h"
|
||||||
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h"
|
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h"
|
||||||
|
|
||||||
namespace latinime {
|
namespace latinime {
|
||||||
|
|
||||||
void DynamicPatriciaTrieNodeReader::fetchNodeInfoFromBufferAndProcessMovedNode(const int nodePos,
|
void DynamicPatriciaTrieNodeReader::fetchNodeInfoFromBufferAndProcessMovedNode(const int nodePos,
|
||||||
const int maxCodePointCount, int *const outCodePoints) {
|
const int maxCodePointCount, int *const outCodePoints) {
|
||||||
const uint8_t *const dictRoot = mBinaryDictionaryInfo->getDictRoot();
|
|
||||||
int pos = nodePos;
|
int pos = nodePos;
|
||||||
mFlags = PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictRoot, &pos);
|
mFlags = PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(mDictRoot, &pos);
|
||||||
const int parentPos =
|
const int parentPos =
|
||||||
DynamicPatriciaTrieReadingUtils::getParentPosAndAdvancePosition(dictRoot, &pos);
|
DynamicPatriciaTrieReadingUtils::getParentPosAndAdvancePosition(mDictRoot, &pos);
|
||||||
mParentPos = (parentPos != 0) ? mNodePos + parentPos : NOT_A_DICT_POS;
|
mParentPos = (parentPos != 0) ? mNodePos + parentPos : NOT_A_DICT_POS;
|
||||||
if (outCodePoints != 0) {
|
if (outCodePoints != 0) {
|
||||||
mCodePointCount = PatriciaTrieReadingUtils::getCharsAndAdvancePosition(
|
mCodePointCount = PatriciaTrieReadingUtils::getCharsAndAdvancePosition(
|
||||||
dictRoot, mFlags, maxCodePointCount, outCodePoints, &pos);
|
mDictRoot, mFlags, maxCodePointCount, outCodePoints, &pos);
|
||||||
} else {
|
} else {
|
||||||
mCodePointCount = PatriciaTrieReadingUtils::skipCharacters(
|
mCodePointCount = PatriciaTrieReadingUtils::skipCharacters(
|
||||||
dictRoot, mFlags, MAX_WORD_LENGTH, &pos);
|
mDictRoot, mFlags, MAX_WORD_LENGTH, &pos);
|
||||||
}
|
}
|
||||||
if (isTerminal()) {
|
if (isTerminal()) {
|
||||||
mProbability = PatriciaTrieReadingUtils::readProbabilityAndAdvancePosition(dictRoot, &pos);
|
mProbability = PatriciaTrieReadingUtils::readProbabilityAndAdvancePosition(mDictRoot, &pos);
|
||||||
} else {
|
} else {
|
||||||
mProbability = NOT_A_PROBABILITY;
|
mProbability = NOT_A_PROBABILITY;
|
||||||
}
|
}
|
||||||
if (hasChildren()) {
|
if (hasChildren()) {
|
||||||
mChildrenPos = DynamicPatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition(
|
mChildrenPos = DynamicPatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition(
|
||||||
dictRoot, mFlags, &pos);
|
mDictRoot, mFlags, &pos);
|
||||||
} else {
|
} else {
|
||||||
mChildrenPos = NOT_A_DICT_POS;
|
mChildrenPos = NOT_A_DICT_POS;
|
||||||
}
|
}
|
||||||
if (PatriciaTrieReadingUtils::hasShortcutTargets(mFlags)) {
|
if (PatriciaTrieReadingUtils::hasShortcutTargets(mFlags)) {
|
||||||
mShortcutPos = pos;
|
mShortcutPos = pos;
|
||||||
BinaryDictionaryTerminalAttributesReadingUtils::skipShortcuts(mBinaryDictionaryInfo, &pos);
|
mShortcutsPolicy->skipAllShortcuts(&pos);
|
||||||
} else {
|
} else {
|
||||||
mShortcutPos = NOT_A_DICT_POS;
|
mShortcutPos = NOT_A_DICT_POS;
|
||||||
}
|
}
|
||||||
|
|
|
@ -25,8 +25,8 @@
|
||||||
|
|
||||||
namespace latinime {
|
namespace latinime {
|
||||||
|
|
||||||
class BinaryDictionaryInfo;
|
|
||||||
class DictionaryBigramsStructurePolicy;
|
class DictionaryBigramsStructurePolicy;
|
||||||
|
class DictionaryShortcutsStructurePolicy;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* This class is used for helping to read nodes of dynamic patricia trie. This class handles moved
|
* This class is used for helping to read nodes of dynamic patricia trie. This class handles moved
|
||||||
|
@ -34,13 +34,14 @@ class DictionaryBigramsStructurePolicy;
|
||||||
*/
|
*/
|
||||||
class DynamicPatriciaTrieNodeReader {
|
class DynamicPatriciaTrieNodeReader {
|
||||||
public:
|
public:
|
||||||
DynamicPatriciaTrieNodeReader(const BinaryDictionaryInfo *const binaryDictionaryInfo,
|
DynamicPatriciaTrieNodeReader(const uint8_t *const dictRoot,
|
||||||
const DictionaryBigramsStructurePolicy *const bigramsPolicy)
|
const DictionaryBigramsStructurePolicy *const bigramsPolicy,
|
||||||
: mBinaryDictionaryInfo(binaryDictionaryInfo), mBigramsPolicy(bigramsPolicy),
|
const DictionaryShortcutsStructurePolicy *const shortcutsPolicy)
|
||||||
mNodePos(NOT_A_VALID_WORD_POS), mFlags(0), mParentPos(NOT_A_DICT_POS),
|
: mDictRoot(dictRoot), mBigramsPolicy(bigramsPolicy),
|
||||||
mCodePointCount(0), mProbability(NOT_A_PROBABILITY), mChildrenPos(NOT_A_DICT_POS),
|
mShortcutsPolicy(shortcutsPolicy), mNodePos(NOT_A_VALID_WORD_POS), mFlags(0),
|
||||||
mShortcutPos(NOT_A_DICT_POS), mBigramPos(NOT_A_DICT_POS),
|
mParentPos(NOT_A_DICT_POS), mCodePointCount(0), mProbability(NOT_A_PROBABILITY),
|
||||||
mSiblingPos(NOT_A_VALID_WORD_POS) {}
|
mChildrenPos(NOT_A_DICT_POS), mShortcutPos(NOT_A_DICT_POS),
|
||||||
|
mBigramPos(NOT_A_DICT_POS), mSiblingPos(NOT_A_VALID_WORD_POS) {}
|
||||||
|
|
||||||
~DynamicPatriciaTrieNodeReader() {}
|
~DynamicPatriciaTrieNodeReader() {}
|
||||||
|
|
||||||
|
@ -120,8 +121,10 @@ class DynamicPatriciaTrieNodeReader {
|
||||||
private:
|
private:
|
||||||
DISALLOW_COPY_AND_ASSIGN(DynamicPatriciaTrieNodeReader);
|
DISALLOW_COPY_AND_ASSIGN(DynamicPatriciaTrieNodeReader);
|
||||||
|
|
||||||
const BinaryDictionaryInfo *const mBinaryDictionaryInfo;
|
// TODO: Consolidate mDictRoot.
|
||||||
|
const uint8_t *const mDictRoot;
|
||||||
const DictionaryBigramsStructurePolicy *const mBigramsPolicy;
|
const DictionaryBigramsStructurePolicy *const mBigramsPolicy;
|
||||||
|
const DictionaryShortcutsStructurePolicy *const mShortcutsPolicy;
|
||||||
int mNodePos;
|
int mNodePos;
|
||||||
DynamicPatriciaTrieReadingUtils::NodeFlags mFlags;
|
DynamicPatriciaTrieReadingUtils::NodeFlags mFlags;
|
||||||
int mParentPos;
|
int mParentPos;
|
||||||
|
|
|
@ -19,7 +19,6 @@
|
||||||
#include "defines.h"
|
#include "defines.h"
|
||||||
#include "suggest/core/dicnode/dic_node.h"
|
#include "suggest/core/dicnode/dic_node.h"
|
||||||
#include "suggest/core/dicnode/dic_node_vector.h"
|
#include "suggest/core/dicnode/dic_node_vector.h"
|
||||||
#include "suggest/core/dictionary/binary_dictionary_info.h"
|
|
||||||
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h"
|
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h"
|
||||||
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h"
|
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h"
|
||||||
#include "suggest/policyimpl/dictionary/patricia_trie_reading_utils.h"
|
#include "suggest/policyimpl/dictionary/patricia_trie_reading_utils.h"
|
||||||
|
@ -34,7 +33,8 @@ void DynamicPatriciaTriePolicy::createAndGetAllChildNodes(const DicNode *const d
|
||||||
if (!dicNode->hasChildren()) {
|
if (!dicNode->hasChildren()) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
DynamicPatriciaTrieNodeReader nodeReader(mBinaryDictionaryInfo, getBigramsStructurePolicy());
|
DynamicPatriciaTrieNodeReader nodeReader(mDictRoot, getBigramsStructurePolicy(),
|
||||||
|
getShortcutsStructurePolicy());
|
||||||
int mergedNodeCodePoints[MAX_WORD_LENGTH];
|
int mergedNodeCodePoints[MAX_WORD_LENGTH];
|
||||||
int nextPos = dicNode->getChildrenPos();
|
int nextPos = dicNode->getChildrenPos();
|
||||||
int totalChildCount = 0;
|
int totalChildCount = 0;
|
||||||
|
@ -79,7 +79,8 @@ int DynamicPatriciaTriePolicy::getCodePointsAndProbabilityAndReturnCodePointCoun
|
||||||
int mergedNodeCodePoints[maxCodePointCount];
|
int mergedNodeCodePoints[maxCodePointCount];
|
||||||
int codePointCount = 0;
|
int codePointCount = 0;
|
||||||
|
|
||||||
DynamicPatriciaTrieNodeReader nodeReader(mBinaryDictionaryInfo, getBigramsStructurePolicy());
|
DynamicPatriciaTrieNodeReader nodeReader(mDictRoot, getBigramsStructurePolicy(),
|
||||||
|
getShortcutsStructurePolicy());
|
||||||
// First, read terminal node and get its probability.
|
// First, read terminal node and get its probability.
|
||||||
nodeReader.fetchNodeInfoFromBufferAndGetNodeCodePoints(nodePos, maxCodePointCount,
|
nodeReader.fetchNodeInfoFromBufferAndGetNodeCodePoints(nodePos, maxCodePointCount,
|
||||||
mergedNodeCodePoints);
|
mergedNodeCodePoints);
|
||||||
|
@ -123,7 +124,8 @@ int DynamicPatriciaTriePolicy::getTerminalNodePositionOfWord(const int *const in
|
||||||
int mergedNodeCodePoints[MAX_WORD_LENGTH];
|
int mergedNodeCodePoints[MAX_WORD_LENGTH];
|
||||||
int currentLength = 0;
|
int currentLength = 0;
|
||||||
int pos = getRootPosition();
|
int pos = getRootPosition();
|
||||||
DynamicPatriciaTrieNodeReader nodeReader(mBinaryDictionaryInfo, getBigramsStructurePolicy());
|
DynamicPatriciaTrieNodeReader nodeReader(mDictRoot, getBigramsStructurePolicy(),
|
||||||
|
getShortcutsStructurePolicy());
|
||||||
while (currentLength <= length) {
|
while (currentLength <= length) {
|
||||||
// When foundMatchedNode becomes true, currentLength is increased at least once.
|
// When foundMatchedNode becomes true, currentLength is increased at least once.
|
||||||
bool foundMatchedNode = false;
|
bool foundMatchedNode = false;
|
||||||
|
@ -194,7 +196,8 @@ int DynamicPatriciaTriePolicy::getUnigramProbability(const int nodePos) const {
|
||||||
if (nodePos == NOT_A_VALID_WORD_POS) {
|
if (nodePos == NOT_A_VALID_WORD_POS) {
|
||||||
return NOT_A_PROBABILITY;
|
return NOT_A_PROBABILITY;
|
||||||
}
|
}
|
||||||
DynamicPatriciaTrieNodeReader nodeReader(mBinaryDictionaryInfo, getBigramsStructurePolicy());
|
DynamicPatriciaTrieNodeReader nodeReader(mDictRoot, getBigramsStructurePolicy(),
|
||||||
|
getShortcutsStructurePolicy());
|
||||||
nodeReader.fetchNodeInfoFromBuffer(nodePos);
|
nodeReader.fetchNodeInfoFromBuffer(nodePos);
|
||||||
if (nodeReader.isDeleted() || nodeReader.isBlacklisted() || nodeReader.isNotAWord()) {
|
if (nodeReader.isDeleted() || nodeReader.isBlacklisted() || nodeReader.isNotAWord()) {
|
||||||
return NOT_A_PROBABILITY;
|
return NOT_A_PROBABILITY;
|
||||||
|
@ -206,7 +209,8 @@ int DynamicPatriciaTriePolicy::getShortcutPositionOfNode(const int nodePos) cons
|
||||||
if (nodePos == NOT_A_VALID_WORD_POS) {
|
if (nodePos == NOT_A_VALID_WORD_POS) {
|
||||||
return NOT_A_DICT_POS;
|
return NOT_A_DICT_POS;
|
||||||
}
|
}
|
||||||
DynamicPatriciaTrieNodeReader nodeReader(mBinaryDictionaryInfo, getBigramsStructurePolicy());
|
DynamicPatriciaTrieNodeReader nodeReader(mDictRoot, getBigramsStructurePolicy(),
|
||||||
|
getShortcutsStructurePolicy());
|
||||||
nodeReader.fetchNodeInfoFromBuffer(nodePos);
|
nodeReader.fetchNodeInfoFromBuffer(nodePos);
|
||||||
if (nodeReader.isDeleted()) {
|
if (nodeReader.isDeleted()) {
|
||||||
return NOT_A_DICT_POS;
|
return NOT_A_DICT_POS;
|
||||||
|
@ -218,7 +222,8 @@ int DynamicPatriciaTriePolicy::getBigramsPositionOfNode(const int nodePos) const
|
||||||
if (nodePos == NOT_A_VALID_WORD_POS) {
|
if (nodePos == NOT_A_VALID_WORD_POS) {
|
||||||
return NOT_A_DICT_POS;
|
return NOT_A_DICT_POS;
|
||||||
}
|
}
|
||||||
DynamicPatriciaTrieNodeReader nodeReader(mBinaryDictionaryInfo, getBigramsStructurePolicy());
|
DynamicPatriciaTrieNodeReader nodeReader(mDictRoot, getBigramsStructurePolicy(),
|
||||||
|
getShortcutsStructurePolicy());
|
||||||
nodeReader.fetchNodeInfoFromBuffer(nodePos);
|
nodeReader.fetchNodeInfoFromBuffer(nodePos);
|
||||||
if (nodeReader.isDeleted()) {
|
if (nodeReader.isDeleted()) {
|
||||||
return NOT_A_DICT_POS;
|
return NOT_A_DICT_POS;
|
||||||
|
|
|
@ -22,19 +22,17 @@
|
||||||
#include "defines.h"
|
#include "defines.h"
|
||||||
#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
|
#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
|
||||||
#include "suggest/policyimpl/dictionary/bigram/bigram_list_policy.h"
|
#include "suggest/policyimpl/dictionary/bigram/bigram_list_policy.h"
|
||||||
|
#include "suggest/policyimpl/dictionary/shortcut/shortcut_list_policy.h"
|
||||||
|
|
||||||
namespace latinime {
|
namespace latinime {
|
||||||
|
|
||||||
class BinaryDictionaryInfo;
|
|
||||||
class DicNode;
|
class DicNode;
|
||||||
class DicNodeVector;
|
class DicNodeVector;
|
||||||
|
|
||||||
class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
|
class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
|
||||||
public:
|
public:
|
||||||
DynamicPatriciaTriePolicy(const uint8_t *const dictRoot,
|
DynamicPatriciaTriePolicy(const uint8_t *const dictRoot)
|
||||||
const BinaryDictionaryInfo *const binaryDictionaryInfo)
|
: mDictRoot(dictRoot), mBigramListPolicy(dictRoot), mShortcutListPolicy(dictRoot) {}
|
||||||
: mDictRoot(dictRoot), mBinaryDictionaryInfo(binaryDictionaryInfo),
|
|
||||||
mBigramListPolicy(dictRoot) {}
|
|
||||||
|
|
||||||
~DynamicPatriciaTriePolicy() {}
|
~DynamicPatriciaTriePolicy() {}
|
||||||
|
|
||||||
|
@ -62,14 +60,18 @@ class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
|
||||||
return &mBigramListPolicy;
|
return &mBigramListPolicy;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const DictionaryShortcutsStructurePolicy *getShortcutsStructurePolicy() const {
|
||||||
|
return &mShortcutListPolicy;
|
||||||
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTriePolicy);
|
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTriePolicy);
|
||||||
static const int MAX_CHILD_COUNT_TO_AVOID_INFINITE_LOOP;
|
static const int MAX_CHILD_COUNT_TO_AVOID_INFINITE_LOOP;
|
||||||
|
|
||||||
|
// TODO: Consolidate mDictRoot.
|
||||||
const uint8_t *const mDictRoot;
|
const uint8_t *const mDictRoot;
|
||||||
// TODO: remove
|
|
||||||
const BinaryDictionaryInfo *const mBinaryDictionaryInfo;
|
|
||||||
const BigramListPolicy mBigramListPolicy;
|
const BigramListPolicy mBigramListPolicy;
|
||||||
|
const ShortcutListPolicy mShortcutListPolicy;
|
||||||
};
|
};
|
||||||
} // namespace latinime
|
} // namespace latinime
|
||||||
#endif // LATINIME_DYNAMIC_PATRICIA_TRIE_POLICY_H
|
#endif // LATINIME_DYNAMIC_PATRICIA_TRIE_POLICY_H
|
||||||
|
|
|
@ -20,8 +20,6 @@
|
||||||
#include "defines.h"
|
#include "defines.h"
|
||||||
#include "suggest/core/dicnode/dic_node.h"
|
#include "suggest/core/dicnode/dic_node.h"
|
||||||
#include "suggest/core/dicnode/dic_node_vector.h"
|
#include "suggest/core/dicnode/dic_node_vector.h"
|
||||||
#include "suggest/core/dictionary/binary_dictionary_info.h"
|
|
||||||
#include "suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.h"
|
|
||||||
#include "suggest/policyimpl/dictionary/binary_format.h"
|
#include "suggest/policyimpl/dictionary/binary_format.h"
|
||||||
#include "suggest/policyimpl/dictionary/patricia_trie_reading_utils.h"
|
#include "suggest/policyimpl/dictionary/patricia_trie_reading_utils.h"
|
||||||
|
|
||||||
|
@ -112,7 +110,7 @@ int PatriciaTriePolicy::getBigramsPositionOfNode(const int nodePos) const {
|
||||||
PatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition(mDictRoot, flags, &pos);
|
PatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition(mDictRoot, flags, &pos);
|
||||||
}
|
}
|
||||||
if (PatriciaTrieReadingUtils::hasShortcutTargets(flags)) {
|
if (PatriciaTrieReadingUtils::hasShortcutTargets(flags)) {
|
||||||
BinaryDictionaryTerminalAttributesReadingUtils::skipShortcuts(mBinaryDictionaryInfo, &pos);
|
mShortcutListPolicy.skipAllShortcuts(&pos);;
|
||||||
}
|
}
|
||||||
return pos;
|
return pos;
|
||||||
}
|
}
|
||||||
|
@ -133,7 +131,7 @@ int PatriciaTriePolicy::createAndGetLeavingChildNode(const DicNode *const dicNod
|
||||||
PatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition(
|
PatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition(
|
||||||
mDictRoot, flags, &pos) : NOT_A_DICT_POS;
|
mDictRoot, flags, &pos) : NOT_A_DICT_POS;
|
||||||
if (PatriciaTrieReadingUtils::hasShortcutTargets(flags)) {
|
if (PatriciaTrieReadingUtils::hasShortcutTargets(flags)) {
|
||||||
BinaryDictionaryTerminalAttributesReadingUtils::skipShortcuts(mBinaryDictionaryInfo, &pos);
|
getShortcutsStructurePolicy()->skipAllShortcuts(&pos);
|
||||||
}
|
}
|
||||||
if (PatriciaTrieReadingUtils::hasBigrams(flags)) {
|
if (PatriciaTrieReadingUtils::hasBigrams(flags)) {
|
||||||
getBigramsStructurePolicy()->skipAllBigrams(&pos);
|
getBigramsStructurePolicy()->skipAllBigrams(&pos);
|
||||||
|
|
|
@ -22,19 +22,17 @@
|
||||||
#include "defines.h"
|
#include "defines.h"
|
||||||
#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
|
#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
|
||||||
#include "suggest/policyimpl/dictionary/bigram/bigram_list_policy.h"
|
#include "suggest/policyimpl/dictionary/bigram/bigram_list_policy.h"
|
||||||
|
#include "suggest/policyimpl/dictionary/shortcut/shortcut_list_policy.h"
|
||||||
|
|
||||||
namespace latinime {
|
namespace latinime {
|
||||||
|
|
||||||
class BinaryDictionaryInfo;
|
|
||||||
class DicNode;
|
class DicNode;
|
||||||
class DicNodeVector;
|
class DicNodeVector;
|
||||||
|
|
||||||
class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
|
class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
|
||||||
public:
|
public:
|
||||||
PatriciaTriePolicy(const uint8_t *const dictRoot,
|
PatriciaTriePolicy(const uint8_t *const dictRoot)
|
||||||
const BinaryDictionaryInfo *const binaryDictionaryInfo)
|
: mDictRoot(dictRoot), mBigramListPolicy(dictRoot), mShortcutListPolicy(dictRoot) {}
|
||||||
: mDictRoot(dictRoot), mBinaryDictionaryInfo(binaryDictionaryInfo),
|
|
||||||
mBigramListPolicy(dictRoot) {}
|
|
||||||
|
|
||||||
~PatriciaTriePolicy() {}
|
~PatriciaTriePolicy() {}
|
||||||
|
|
||||||
|
@ -62,13 +60,16 @@ class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
|
||||||
return &mBigramListPolicy;
|
return &mBigramListPolicy;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const DictionaryShortcutsStructurePolicy *getShortcutsStructurePolicy() const {
|
||||||
|
return &mShortcutListPolicy;
|
||||||
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
DISALLOW_IMPLICIT_CONSTRUCTORS(PatriciaTriePolicy);
|
DISALLOW_IMPLICIT_CONSTRUCTORS(PatriciaTriePolicy);
|
||||||
|
|
||||||
const uint8_t *const mDictRoot;
|
const uint8_t *const mDictRoot;
|
||||||
// TODO: remove
|
|
||||||
const BinaryDictionaryInfo *const mBinaryDictionaryInfo;
|
|
||||||
const BigramListPolicy mBigramListPolicy;
|
const BigramListPolicy mBigramListPolicy;
|
||||||
|
const ShortcutListPolicy mShortcutListPolicy;
|
||||||
|
|
||||||
int createAndGetLeavingChildNode(const DicNode *const dicNode, const int nodePos,
|
int createAndGetLeavingChildNode(const DicNode *const dicNode, const int nodePos,
|
||||||
const NodeFilter *const nodeFilter, DicNodeVector *const childDicNodes) const;
|
const NodeFilter *const nodeFilter, DicNodeVector *const childDicNodes) const;
|
||||||
|
|
|
@ -0,0 +1,75 @@
|
||||||
|
/*
|
||||||
|
* Copyright (C) 2013 The Android Open Source Project
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#ifndef LATINIME_SHORTCUT_LIST_POLICY_H
|
||||||
|
#define LATINIME_SHORTCUT_LIST_POLICY_H
|
||||||
|
|
||||||
|
#include <stdint.h>
|
||||||
|
|
||||||
|
#include "defines.h"
|
||||||
|
#include "suggest/core/policy/dictionary_shortcuts_structure_policy.h"
|
||||||
|
// TODO: Move shortcuts reading methods to policyimpl.
|
||||||
|
#include "suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.h"
|
||||||
|
|
||||||
|
namespace latinime {
|
||||||
|
|
||||||
|
class ShortcutListPolicy : public DictionaryShortcutsStructurePolicy {
|
||||||
|
public:
|
||||||
|
explicit ShortcutListPolicy(const uint8_t *const shortcutBuf)
|
||||||
|
: mShortcutsBuf(shortcutBuf) {}
|
||||||
|
|
||||||
|
~ShortcutListPolicy() {}
|
||||||
|
|
||||||
|
int getStartPos(const int pos) const {
|
||||||
|
int listPos = pos;
|
||||||
|
BinaryDictionaryTerminalAttributesReadingUtils::getShortcutListSizeAndForwardPointer(
|
||||||
|
mShortcutsBuf, &listPos);
|
||||||
|
return listPos;
|
||||||
|
}
|
||||||
|
|
||||||
|
void getNextShortcut(const int maxCodePointCount, int *const outCodePoint,
|
||||||
|
int *const outCodePointCount, bool *const outIsWhitelist, bool *const outHasNext,
|
||||||
|
int *const pos) const {
|
||||||
|
const BinaryDictionaryTerminalAttributesReadingUtils::ShortcutFlags flags =
|
||||||
|
BinaryDictionaryTerminalAttributesReadingUtils::getFlagsAndForwardPointer(
|
||||||
|
mShortcutsBuf, pos);
|
||||||
|
if (outHasNext) {
|
||||||
|
*outHasNext = BinaryDictionaryTerminalAttributesReadingUtils::hasNext(flags);
|
||||||
|
}
|
||||||
|
if (outIsWhitelist) {
|
||||||
|
*outIsWhitelist =
|
||||||
|
BinaryDictionaryTerminalAttributesReadingUtils::isWhitelist(flags);
|
||||||
|
}
|
||||||
|
if (outCodePoint) {
|
||||||
|
*outCodePointCount =
|
||||||
|
BinaryDictionaryTerminalAttributesReadingUtils::readShortcutTarget(
|
||||||
|
mShortcutsBuf, maxCodePointCount, outCodePoint, pos);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void skipAllShortcuts(int *const pos) const {
|
||||||
|
const int shortcutListSize = BinaryDictionaryTerminalAttributesReadingUtils
|
||||||
|
::getShortcutListSizeAndForwardPointer(mShortcutsBuf, pos);
|
||||||
|
*pos += shortcutListSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
private:
|
||||||
|
DISALLOW_IMPLICIT_CONSTRUCTORS(ShortcutListPolicy);
|
||||||
|
|
||||||
|
const uint8_t *const mShortcutsBuf;
|
||||||
|
};
|
||||||
|
} // namespace latinime
|
||||||
|
#endif // LATINIME_SHORTCUT_LIST_POLICY_H
|
Loading…
Reference in a new issue