nyancrimew/LatinIME
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge "Move bigram reading methods to BigramListReadingUtils."

Browse source
This commit is contained in:
Keisuke Kuroyanagi 2013-08-12 03:26:27 +00:00 committed by Android (Google) Code Review
commit 53830bf464
10 changed files with 161 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
native/jni/Android.mk
View file

@ -71,6 +71,7 @@ LATIN_IME_CORE_SRC_FILES := \
suggest/core/policy/weighting.cpp \
suggest/core/session/dic_traverse_session.cpp \
$(addprefix suggest/policyimpl/dictionary/, \
bigram/bigram_list_reading_utils.cpp \
dictionary_structure_with_buffer_policy_factory.cpp \
dynamic_patricia_trie_node_reader.cpp \
dynamic_patricia_trie_policy.cpp \

23
native/jni/src/suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.cpp
View file

@ -36,27 +36,4 @@ const int TaUtils::SHORTCUT_LIST_SIZE_FIELD_SIZE = 2;
// The numeric value of the shortcut probability that means 'whitelist'.
const int TaUtils::WHITELIST_SHORTCUT_PROBABILITY = 15;
/* static */ int TaUtils::getBigramAddressAndForwardPointer(
const uint8_t *const dictRoot, const TerminalAttributeFlags flags,
int *const pos) {
int offset = 0;
const int origin = *pos;
switch (MASK_ATTRIBUTE_ADDRESS_TYPE & flags) {
case FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE:
offset = ByteArrayUtils::readUint8AndAdvancePosition(dictRoot, pos);
break;
case FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES:
offset = ByteArrayUtils::readUint16AndAdvancePosition(dictRoot, pos);
break;
case FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES:
offset = ByteArrayUtils::readUint24AndAdvancePosition(dictRoot, pos);
break;
}
if (isOffsetNegative(flags)) {
return origin - offset;
} else {
return origin + offset;
}
}
} // namespace latinime

15
native/jni/src/suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.h
View file

@ -28,7 +28,6 @@ namespace latinime {
class BinaryDictionaryTerminalAttributesReadingUtils {
public:
typedef uint8_t TerminalAttributeFlags;
typedef TerminalAttributeFlags BigramFlags;
typedef TerminalAttributeFlags ShortcutFlags;
static AK_FORCE_INLINE TerminalAttributeFlags getFlagsAndForwardPointer(
@ -44,20 +43,6 @@ class BinaryDictionaryTerminalAttributesReadingUtils {
return (flags & FLAG_ATTRIBUTE_HAS_NEXT) != 0;
}
// Bigrams reading methods
static AK_FORCE_INLINE void skipExistingBigrams(
const uint8_t *const dictRoot, int *const pos) {
BigramFlags flags = getFlagsAndForwardPointer(dictRoot, pos);
while (hasNext(flags)) {
*pos += attributeAddressSize(flags);
flags = getFlagsAndForwardPointer(dictRoot, pos);
}
*pos += attributeAddressSize(flags);
}
static int getBigramAddressAndForwardPointer(
const uint8_t *const dictRoot, const BigramFlags flags, int *const pos);
// Shortcuts reading methods
// This method returns the size of the shortcut list region excluding the shortcut list size
// field at the beginning.

18
native/jni/src/suggest/policyimpl/dictionary/bigrams/bigram_list_policy.h → native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_policy.h
View file

@ -21,8 +21,7 @@
#include "defines.h"
#include "suggest/core/policy/dictionary_bigrams_structure_policy.h"
// TODO: Move bigrams reading methods to policyimpl.
#include "suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.h"
#include "suggest/policyimpl/dictionary/bigram/bigram_list_reading_utils.h"
namespace latinime {
@ -34,19 +33,16 @@ class BigramListPolicy : public DictionaryBigramsStructurePolicy {
void getNextBigram(int *const outBigramPos, int *const outProbability, bool *const outHasNext,
int *const pos) const {
const BinaryDictionaryTerminalAttributesReadingUtils::BigramFlags flags =
BinaryDictionaryTerminalAttributesReadingUtils::getFlagsAndForwardPointer(
mBigramsBuf, pos);
*outBigramPos =
BinaryDictionaryTerminalAttributesReadingUtils::getBigramAddressAndForwardPointer(
const BigramListReadingUtils::BigramFlags flags =
BigramListReadingUtils::getFlagsAndForwardPointer(mBigramsBuf, pos);
*outBigramPos = BigramListReadingUtils::getBigramAddressAndForwardPointer(
mBigramsBuf, flags, pos);
*outProbability =
BinaryDictionaryTerminalAttributesReadingUtils::getProbabilityFromFlags(flags);
*outHasNext = BinaryDictionaryTerminalAttributesReadingUtils::hasNext(flags);
*outProbability = BigramListReadingUtils::getProbabilityFromFlags(flags);
*outHasNext = BigramListReadingUtils::hasNext(flags);
}
void skipAllBigrams(int *const pos) const {
BinaryDictionaryTerminalAttributesReadingUtils::skipExistingBigrams(mBigramsBuf, pos);
BigramListReadingUtils::skipExistingBigrams(mBigramsBuf, pos);
}
private:

62
native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_reading_utils.cpp Normal file
View file

@ -0,0 +1,62 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "suggest/policyimpl/dictionary/bigram/bigram_list_reading_utils.h"
#include "suggest/core/dictionary/byte_array_utils.h"
namespace latinime {
const BigramListReadingUtils::BigramFlags BigramListReadingUtils::MASK_ATTRIBUTE_ADDRESS_TYPE =
0x30;
const BigramListReadingUtils::BigramFlags
BigramListReadingUtils::FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE = 0x10;
const BigramListReadingUtils::BigramFlags
BigramListReadingUtils::FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES = 0x20;
const BigramListReadingUtils::BigramFlags
BigramListReadingUtils::FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES = 0x30;
const BigramListReadingUtils::BigramFlags
BigramListReadingUtils::FLAG_ATTRIBUTE_OFFSET_NEGATIVE = 0x40;
// Flag for presence of more attributes
const BigramListReadingUtils::BigramFlags BigramListReadingUtils::FLAG_ATTRIBUTE_HAS_NEXT = 0x80;
// Mask for attribute probability, stored on 4 bits inside the flags byte.
const BigramListReadingUtils::BigramFlags
BigramListReadingUtils::MASK_ATTRIBUTE_PROBABILITY = 0x0F;
const int BigramListReadingUtils::ATTRIBUTE_ADDRESS_SHIFT = 4;
/* static */ int BigramListReadingUtils::getBigramAddressAndForwardPointer(
const uint8_t *const bigramsBuf, const BigramFlags flags, int *const pos) {
int offset = 0;
const int origin = *pos;
switch (MASK_ATTRIBUTE_ADDRESS_TYPE & flags) {
case FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE:
offset = ByteArrayUtils::readUint8AndAdvancePosition(bigramsBuf, pos);
break;
case FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES:
offset = ByteArrayUtils::readUint16AndAdvancePosition(bigramsBuf, pos);
break;
case FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES:
offset = ByteArrayUtils::readUint24AndAdvancePosition(bigramsBuf, pos);
break;
}
if (isOffsetNegative(flags)) {
return origin - offset;
} else {
return origin + offset;
}
}
} // namespace latinime

88
native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_reading_utils.h Normal file
View file

@ -0,0 +1,88 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef LATINIME_BIGRAM_LIST_READING_UTILS_H
#define LATINIME_BIGRAM_LIST_READING_UTILS_H
#include <stdint.h>
#include "defines.h"
#include "suggest/core/dictionary/byte_array_utils.h"
namespace latinime {
class BigramListReadingUtils {
public:
typedef uint8_t BigramFlags;
static AK_FORCE_INLINE BigramFlags getFlagsAndForwardPointer(
const uint8_t *const bigramsBuf, int *const pos) {
return ByteArrayUtils::readUint8AndAdvancePosition(bigramsBuf, pos);
}
static AK_FORCE_INLINE int getProbabilityFromFlags(const BigramFlags flags) {
return flags & MASK_ATTRIBUTE_PROBABILITY;
}
static AK_FORCE_INLINE bool hasNext(const BigramFlags flags) {
return (flags & FLAG_ATTRIBUTE_HAS_NEXT) != 0;
}
// Bigrams reading methods
static AK_FORCE_INLINE void skipExistingBigrams(const uint8_t *const bigramsBuf,
int *const pos) {
BigramFlags flags = getFlagsAndForwardPointer(bigramsBuf, pos);
while (hasNext(flags)) {
*pos += attributeAddressSize(flags);
flags = getFlagsAndForwardPointer(bigramsBuf, pos);
}
*pos += attributeAddressSize(flags);
}
static int getBigramAddressAndForwardPointer(const uint8_t *const bigramsBuf,
const BigramFlags flags, int *const pos);
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(BigramListReadingUtils);
static const BigramFlags MASK_ATTRIBUTE_ADDRESS_TYPE;
static const BigramFlags FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE;
static const BigramFlags FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES;
static const BigramFlags FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES;
static const BigramFlags FLAG_ATTRIBUTE_OFFSET_NEGATIVE;
static const BigramFlags FLAG_ATTRIBUTE_HAS_NEXT;
static const BigramFlags MASK_ATTRIBUTE_PROBABILITY;
static const int ATTRIBUTE_ADDRESS_SHIFT;
static AK_FORCE_INLINE bool isOffsetNegative(const BigramFlags flags) {
return (flags & FLAG_ATTRIBUTE_OFFSET_NEGATIVE) != 0;
}
static AK_FORCE_INLINE int attributeAddressSize(const BigramFlags flags) {
return (flags & MASK_ATTRIBUTE_ADDRESS_TYPE) >> ATTRIBUTE_ADDRESS_SHIFT;
/* Note: this is a value-dependant optimization of what may probably be
more readably written this way:
switch (flags * BinaryFormat::MASK_ATTRIBUTE_ADDRESS_TYPE) {
case FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: return 1;
case FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: return 2;
case FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTE: return 3;
default: return 0;
}
*/
}
};
} // namespace latinime
#endif // LATINIME_BIGRAM_LIST_READING_UTILS_H

2
native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp
View file

@ -18,7 +18,7 @@
#include "suggest/core/dictionary/binary_dictionary_info.h"
#include "suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.h"
#include "suggest/policyimpl/dictionary/bigrams/bigram_list_policy.h"
#include "suggest/core/policy/dictionary_bigrams_structure_policy.h"
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h"
namespace latinime {

1
native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h
View file

@ -20,7 +20,6 @@
#include <stdint.h>
#include "defines.h"
#include "suggest/policyimpl/dictionary/bigrams/bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/patricia_trie_reading_utils.h"

2
native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h
View file

@ -21,7 +21,7 @@
#include "defines.h"
#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
#include "suggest/policyimpl/dictionary/bigrams/bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/bigram/bigram_list_policy.h"
namespace latinime {

2
native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h
View file

@ -21,7 +21,7 @@
#include "defines.h"
#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
#include "suggest/policyimpl/dictionary/bigrams/bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/bigram/bigram_list_policy.h"
namespace latinime {