nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Move shortcut reading methods. 

Moved form BinaryFormat to BinaryDictionaryTerminalAttributesReadingUtils.

Bug: 6669677

Change-Id: Ia1ab25854effbf61df37837fe26755ac7dc4d020
main
Keisuke Kuroynagi 2013-06-26 14:41:22 +09:00
parent 4da287d0d1
commit 5ac44bdc2a
5 changed files with 77 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
native/jni/src/suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.cpp
View File

@ -33,6 +33,9 @@ const TaUtils::TerminalAttributeFlags TaUtils::FLAG_ATTRIBUTE_HAS_NEXT = 0x80;
// Mask for attribute probability, stored on 4 bits inside the flags byte.
const TaUtils::TerminalAttributeFlags TaUtils::MASK_ATTRIBUTE_PROBABILITY = 0x0F;
const int TaUtils::ATTRIBUTE_ADDRESS_SHIFT = 4;
const int TaUtils::SHORTCUT_LIST_SIZE_FIELD_SIZE = 2;
// The numeric value of the shortcut probability that means 'whitelist'.
const int TaUtils::WHITELIST_SHORTCUT_PROBABILITY = 15;
/* static */ int TaUtils::getBigramAddressAndForwardPointer(
const BinaryDictionaryInfo *const binaryDictionaryInfo, const TerminalAttributeFlags flags,

31
native/jni/src/suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.h
View File

@ -29,6 +29,7 @@ class BinaryDictionaryTerminalAttributesReadingUtils {
public:
typedef uint8_t TerminalAttributeFlags;
typedef TerminalAttributeFlags BigramFlags;
typedef TerminalAttributeFlags ShortcutFlags;
static AK_FORCE_INLINE TerminalAttributeFlags getFlagsAndForwardPointer(
const BinaryDictionaryInfo *const binaryDictionaryInfo, int *const pos) {
@ -59,6 +60,34 @@ class BinaryDictionaryTerminalAttributesReadingUtils {
const BinaryDictionaryInfo *const binaryDictionaryInfo, const BigramFlags flags,
int *const pos);
// Shortcuts reading methods
// This method returns the size of the shortcut list region excluding the shortcut list size
// field at the beginning.
static AK_FORCE_INLINE int getShortcutListSizeAndForwardPointer(
const BinaryDictionaryInfo *const binaryDictionaryInfo, int *const pos) {
// readUint16andAdvancePosition() returns an offset *including* the uint16 field itself.
return ByteArrayUtils::readUint16andAdvancePosition(
binaryDictionaryInfo->getDictRoot(), pos) - SHORTCUT_LIST_SIZE_FIELD_SIZE;
}
static AK_FORCE_INLINE void skipShortcuts(
const BinaryDictionaryInfo *const binaryDictionaryInfo, int *const pos) {
const int shortcutListSize = getShortcutListSizeAndForwardPointer(
binaryDictionaryInfo, pos);
*pos += shortcutListSize;
}
static AK_FORCE_INLINE bool isWhitelist(const ShortcutFlags flags) {
return getProbabilityFromFlags(flags) == WHITELIST_SHORTCUT_PROBABILITY;
}
static AK_FORCE_INLINE int readShortcutTarget(
const BinaryDictionaryInfo *const binaryDictionaryInfo, const int maxLength,
int *const outWord, int *const pos) {
return ByteArrayUtils::readStringAndAdvancePosition(
binaryDictionaryInfo->getDictRoot(), maxLength, outWord, pos);
}
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(BinaryDictionaryTerminalAttributesReadingUtils);
@ -70,6 +99,8 @@ class BinaryDictionaryTerminalAttributesReadingUtils {
static const TerminalAttributeFlags FLAG_ATTRIBUTE_HAS_NEXT;
static const TerminalAttributeFlags MASK_ATTRIBUTE_PROBABILITY;
static const int ATTRIBUTE_ADDRESS_SHIFT;
static const int SHORTCUT_LIST_SIZE_FIELD_SIZE;
static const int WHITELIST_SHORTCUT_PROBABILITY;
static AK_FORCE_INLINE bool isOffsetNegative(const TerminalAttributeFlags flags) {
return (flags & FLAG_ATTRIBUTE_OFFSET_NEGATIVE) != 0;

39
native/jni/src/suggest/core/dictionary/binary_format.h
View File

@ -52,14 +52,10 @@ class BinaryFormat {
// Mask for attribute probability, stored on 4 bits inside the flags byte.
static const int MASK_ATTRIBUTE_PROBABILITY = 0x0F;
// The numeric value of the shortcut probability that means 'whitelist'.
static const int WHITELIST_SHORTCUT_PROBABILITY = 15;
// Mask and flags for attribute address type selection.
static const int MASK_ATTRIBUTE_ADDRESS_TYPE = 0x30;
static const int SHORTCUT_LIST_SIZE_SIZE = 2;
static bool hasBlacklistedOrNotAWordFlag(const int flags);
static int getGroupCountAndForwardPointer(const uint8_t *const dict, int *pos);
static uint8_t getFlagsAndForwardPointer(const uint8_t *const dict, int *pos);
@ -73,9 +69,6 @@ class BinaryFormat {
const int pos);
static int readChildrenPosition(const uint8_t *const dict, const uint8_t flags, const int pos);
static bool hasChildrenInFlags(const uint8_t flags);
static int getAttributeAddressAndForwardPointer(const uint8_t *const dict, const uint8_t flags,
int *pos);
static int getAttributeProbabilityFromFlags(const int flags);
static int getTerminalPosition(const uint8_t *const root, const int *const inWord,
const int length, const bool forceLowerCaseSearch);
static int getWordAtAddress(const uint8_t *const root, const int address, const int maxDepth,
@ -260,38 +253,6 @@ inline bool BinaryFormat::hasChildrenInFlags(const uint8_t flags) {
return (FLAG_GROUP_ADDRESS_TYPE_NOADDRESS != (MASK_GROUP_ADDRESS_TYPE & flags));
}
AK_FORCE_INLINE int BinaryFormat::getAttributeAddressAndForwardPointer(const uint8_t *const dict,
const uint8_t flags, int *pos) {
int offset = 0;
const int origin = *pos;
switch (MASK_ATTRIBUTE_ADDRESS_TYPE & flags) {
case FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE:
offset = dict[origin];
*pos = origin + 1;
break;
case FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES:
offset = dict[origin] << 8;
offset += dict[origin + 1];
*pos = origin + 2;
break;
case FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES:
offset = dict[origin] << 16;
offset += dict[origin + 1] << 8;
offset += dict[origin + 2];
*pos = origin + 3;
break;
}
if (FLAG_ATTRIBUTE_OFFSET_NEGATIVE & flags) {
return origin - offset;
} else {
return origin + offset;
}
}
inline int BinaryFormat::getAttributeProbabilityFromFlags(const int flags) {
return flags & MASK_ATTRIBUTE_PROBABILITY;
}
// This function gets the byte position of the last chargroup of the exact matching word in the
// dictionary. If no match is found, it returns NOT_VALID_WORD.
AK_FORCE_INLINE int BinaryFormat::getTerminalPosition(const uint8_t *const root,

12
native/jni/src/suggest/core/dictionary/shortcut_utils.h
View File

@ -29,15 +29,15 @@ class ShortcutUtils {
int outputWordIndex, const int finalScore, int *const outputCodePoints,
int *const frequencies, int *const outputTypes, const bool sameAsTyped) {
TerminalAttributes::ShortcutIterator iterator = terminalAttributes->getShortcutIterator();
while (iterator.hasNextShortcutTarget() && outputWordIndex < MAX_RESULTS) {
int shortcutTarget[MAX_WORD_LENGTH];
int shortcutProbability;
const int shortcutTargetStringLength = iterator.getNextShortcutTarget(
MAX_WORD_LENGTH, shortcutTarget, &shortcutProbability);
while (iterator.hasNextShortcutTarget() && outputWordIndex < MAX_RESULTS) {
bool isWhilelist;
int shortcutTargetStringLength;
iterator.nextShortcutTarget(MAX_WORD_LENGTH, shortcutTarget,
&shortcutTargetStringLength, &isWhilelist);
int shortcutScore;
int kind;
if (shortcutProbability == BinaryFormat::WHITELIST_SHORTCUT_PROBABILITY
&& sameAsTyped) {
if (isWhilelist && sameAsTyped) {
shortcutScore = S_INT_MAX;
kind = Dictionary::KIND_WHITELIST;
} else {

67
native/jni/src/suggest/core/dictionary/terminal_attributes.h
View File

@ -20,6 +20,7 @@
#include <stdint.h>
#include "suggest/core/dictionary/binary_dictionary_info.h"
#include "suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.h"
#include "suggest/core/dictionary/binary_format.h"
namespace latinime {
@ -33,60 +34,66 @@ class TerminalAttributes {
public:
class ShortcutIterator {
public:
ShortcutIterator(const BinaryDictionaryInfo *const binaryDictionaryInfo, const int pos,
const uint8_t flags)
: mBinaryDicitionaryInfo(binaryDictionaryInfo), mPos(pos),
mHasNextShortcutTarget(0 != (flags & BinaryFormat::FLAG_HAS_SHORTCUT_TARGETS)) {
}
ShortcutIterator(const BinaryDictionaryInfo *const binaryDictionaryInfo,
const int shortcutPos, const bool hasShortcutList)
: mBinaryDictionaryInfo(binaryDictionaryInfo), mPos(shortcutPos),
mHasNextShortcutTarget(hasShortcutList) {}
inline bool hasNextShortcutTarget() const {
return mHasNextShortcutTarget;
}
// Gets the shortcut target itself as an int string. For parameters and return value
// see BinaryFormat::getWordAtAddress.
inline int getNextShortcutTarget(const int maxDepth, int *outWord, int *outFreq) {
const int shortcutFlags = BinaryFormat::getFlagsAndForwardPointer(
mBinaryDicitionaryInfo->getDictRoot(), &mPos);
mHasNextShortcutTarget = 0 != (shortcutFlags & BinaryFormat::FLAG_ATTRIBUTE_HAS_NEXT);
unsigned int i;
for (i = 0; i < MAX_WORD_LENGTH; ++i) {
const int codePoint = BinaryFormat::getCodePointAndForwardPointer(
mBinaryDicitionaryInfo->getDictRoot(), &mPos);
if (NOT_A_CODE_POINT == codePoint) break;
outWord[i] = codePoint;
// Gets the shortcut target itself as an int string and put it to outTarget, put its length
// to outTargetLength, put whether it is whitelist to outIsWhitelist.
AK_FORCE_INLINE void nextShortcutTarget(
const int maxDepth, int *const outTarget, int *const outTargetLength,
bool *const outIsWhitelist) {
const BinaryDictionaryTerminalAttributesReadingUtils::ShortcutFlags flags =
BinaryDictionaryTerminalAttributesReadingUtils::getFlagsAndForwardPointer(
mBinaryDictionaryInfo, &mPos);
mHasNextShortcutTarget =
BinaryDictionaryTerminalAttributesReadingUtils::hasNext(flags);
if (outIsWhitelist) {
*outIsWhitelist =
BinaryDictionaryTerminalAttributesReadingUtils::isWhitelist(flags);
}
if (outTargetLength) {
*outTargetLength =
BinaryDictionaryTerminalAttributesReadingUtils::readShortcutTarget(
mBinaryDictionaryInfo, maxDepth, outTarget, &mPos);
}
*outFreq = BinaryFormat::getAttributeProbabilityFromFlags(shortcutFlags);
return i;
}
private:
const BinaryDictionaryInfo *const mBinaryDicitionaryInfo;
const BinaryDictionaryInfo *const mBinaryDictionaryInfo;
int mPos;
bool mHasNextShortcutTarget;
};
TerminalAttributes(const BinaryDictionaryInfo *const binaryDicitonaryInfo,
const uint8_t flags, const int pos)
: mBinaryDicitionaryInfo(binaryDicitonaryInfo), mFlags(flags), mStartPos(pos) {
}
TerminalAttributes(const BinaryDictionaryInfo *const binaryDictionaryInfo,
const uint8_t nodeFlags, const int shortcutPos)
: mBinaryDictionaryInfo(binaryDictionaryInfo),
mNodeFlags(nodeFlags), mShortcutListSizePos(shortcutPos) {}
inline ShortcutIterator getShortcutIterator() const {
// The size of the shortcuts is stored here so that the whole shortcut chunk can be
// skipped quickly, so we ignore it.
return ShortcutIterator(
mBinaryDicitionaryInfo, mStartPos + BinaryFormat::SHORTCUT_LIST_SIZE_SIZE, mFlags);
int shortcutPos = mShortcutListSizePos;
BinaryDictionaryTerminalAttributesReadingUtils::getShortcutListSizeAndForwardPointer(
mBinaryDictionaryInfo, &shortcutPos);
const bool hasShortcutList = 0 != (mNodeFlags & BinaryFormat::FLAG_HAS_SHORTCUT_TARGETS);
return ShortcutIterator(mBinaryDictionaryInfo, shortcutPos, hasShortcutList);
}
bool isBlacklistedOrNotAWord() const {
return BinaryFormat::hasBlacklistedOrNotAWordFlag(mFlags);
return BinaryFormat::hasBlacklistedOrNotAWordFlag(mNodeFlags);
}
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(TerminalAttributes);
const BinaryDictionaryInfo *const mBinaryDicitionaryInfo;
const uint8_t mFlags;
const int mStartPos;
const BinaryDictionaryInfo *const mBinaryDictionaryInfo;
const uint8_t mNodeFlags;
const int mShortcutListSizePos;
};
} // namespace latinime
#endif // LATINIME_TERMINAL_ATTRIBUTES_H