nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Move bigram reading methods to BigramListReadingUtils. 

Bug: 6669677
Change-Id: Icb024a8af577323b4dcf74f4c4054cfb8b57a943
main
Keisuke Kuroyanagi 2013-08-12 11:54:31 +09:00
parent 4a2f6a2216
commit 277053af7c
10 changed files with 161 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
native/jni/Android.mk
View File

@ -71,6 +71,7 @@ LATIN_IME_CORE_SRC_FILES := \
suggest/core/policy/weighting.cpp \
suggest/core/session/dic_traverse_session.cpp \
$(addprefix suggest/policyimpl/dictionary/, \
bigram/bigram_list_reading_utils.cpp \
dictionary_structure_with_buffer_policy_factory.cpp \
dynamic_patricia_trie_node_reader.cpp \
dynamic_patricia_trie_policy.cpp \

23
native/jni/src/suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.cpp
View File

@ -36,27 +36,4 @@ const int TaUtils::SHORTCUT_LIST_SIZE_FIELD_SIZE = 2;
// The numeric value of the shortcut probability that means 'whitelist'.
const int TaUtils::WHITELIST_SHORTCUT_PROBABILITY = 15;
/* static */ int TaUtils::getBigramAddressAndForwardPointer(
const uint8_t *const dictRoot, const TerminalAttributeFlags flags,
int *const pos) {
int offset = 0;
const int origin = *pos;
switch (MASK_ATTRIBUTE_ADDRESS_TYPE & flags) {
case FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE:
offset = ByteArrayUtils::readUint8AndAdvancePosition(dictRoot, pos);
break;
case FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES:
offset = ByteArrayUtils::readUint16AndAdvancePosition(dictRoot, pos);
break;
case FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES:
offset = ByteArrayUtils::readUint24AndAdvancePosition(dictRoot, pos);
break;
}
if (isOffsetNegative(flags)) {
return origin - offset;
} else {
return origin + offset;
}
}
} // namespace latinime

15
native/jni/src/suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.h
View File

@ -28,7 +28,6 @@ namespace latinime {
class BinaryDictionaryTerminalAttributesReadingUtils {
public:
typedef uint8_t TerminalAttributeFlags;
typedef TerminalAttributeFlags BigramFlags;
typedef TerminalAttributeFlags ShortcutFlags;
static AK_FORCE_INLINE TerminalAttributeFlags getFlagsAndForwardPointer(
@ -44,20 +43,6 @@ class BinaryDictionaryTerminalAttributesReadingUtils {
return (flags & FLAG_ATTRIBUTE_HAS_NEXT) != 0;
}
// Bigrams reading methods
static AK_FORCE_INLINE void skipExistingBigrams(
const uint8_t *const dictRoot, int *const pos) {
BigramFlags flags = getFlagsAndForwardPointer(dictRoot, pos);
while (hasNext(flags)) {
*pos += attributeAddressSize(flags);
flags = getFlagsAndForwardPointer(dictRoot, pos);
}
*pos += attributeAddressSize(flags);
}
static int getBigramAddressAndForwardPointer(
const uint8_t *const dictRoot, const BigramFlags flags, int *const pos);
// Shortcuts reading methods
// This method returns the size of the shortcut list region excluding the shortcut list size
// field at the beginning.

18
native/jni/src/suggest/policyimpl/dictionary/bigrams/bigram_list_policy.h → native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_policy.h
View File

@ -21,8 +21,7 @@
#include "defines.h"
#include "suggest/core/policy/dictionary_bigrams_structure_policy.h"
// TODO: Move bigrams reading methods to policyimpl.
#include "suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.h"
#include "suggest/policyimpl/dictionary/bigram/bigram_list_reading_utils.h"
namespace latinime {
@ -34,19 +33,16 @@ class BigramListPolicy : public DictionaryBigramsStructurePolicy {
void getNextBigram(int *const outBigramPos, int *const outProbability, bool *const outHasNext,
int *const pos) const {
const BinaryDictionaryTerminalAttributesReadingUtils::BigramFlags flags =
BinaryDictionaryTerminalAttributesReadingUtils::getFlagsAndForwardPointer(
mBigramsBuf, pos);
*outBigramPos =
BinaryDictionaryTerminalAttributesReadingUtils::getBigramAddressAndForwardPointer(
const BigramListReadingUtils::BigramFlags flags =
BigramListReadingUtils::getFlagsAndForwardPointer(mBigramsBuf, pos);
*outBigramPos = BigramListReadingUtils::getBigramAddressAndForwardPointer(
mBigramsBuf, flags, pos);
*outProbability =
BinaryDictionaryTerminalAttributesReadingUtils::getProbabilityFromFlags(flags);
*outHasNext = BinaryDictionaryTerminalAttributesReadingUtils::hasNext(flags);
*outProbability = BigramListReadingUtils::getProbabilityFromFlags(flags);
*outHasNext = BigramListReadingUtils::hasNext(flags);
}
void skipAllBigrams(int *const pos) const {
BinaryDictionaryTerminalAttributesReadingUtils::skipExistingBigrams(mBigramsBuf, pos);
BigramListReadingUtils::skipExistingBigrams(mBigramsBuf, pos);
}
private:

62
native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_reading_utils.cpp Normal file
View File

@ -0,0 +1,62 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "suggest/policyimpl/dictionary/bigram/bigram_list_reading_utils.h"
#include "suggest/core/dictionary/byte_array_utils.h"
namespace latinime {
const BigramListReadingUtils::BigramFlags BigramListReadingUtils::MASK_ATTRIBUTE_ADDRESS_TYPE =
0x30;
const BigramListReadingUtils::BigramFlags
BigramListReadingUtils::FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE = 0x10;
const BigramListReadingUtils::BigramFlags
BigramListReadingUtils::FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES = 0x20;
const BigramListReadingUtils::BigramFlags
BigramListReadingUtils::FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES = 0x30;
const BigramListReadingUtils::BigramFlags
BigramListReadingUtils::FLAG_ATTRIBUTE_OFFSET_NEGATIVE = 0x40;
// Flag for presence of more attributes
const BigramListReadingUtils::BigramFlags BigramListReadingUtils::FLAG_ATTRIBUTE_HAS_NEXT = 0x80;
// Mask for attribute probability, stored on 4 bits inside the flags byte.
const BigramListReadingUtils::BigramFlags
BigramListReadingUtils::MASK_ATTRIBUTE_PROBABILITY = 0x0F;
const int BigramListReadingUtils::ATTRIBUTE_ADDRESS_SHIFT = 4;
/* static */ int BigramListReadingUtils::getBigramAddressAndForwardPointer(
const uint8_t *const bigramsBuf, const BigramFlags flags, int *const pos) {
int offset = 0;
const int origin = *pos;
switch (MASK_ATTRIBUTE_ADDRESS_TYPE & flags) {
case FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE:
offset = ByteArrayUtils::readUint8AndAdvancePosition(bigramsBuf, pos);
break;
case FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES:
offset = ByteArrayUtils::readUint16AndAdvancePosition(bigramsBuf, pos);
break;
case FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES:
offset = ByteArrayUtils::readUint24AndAdvancePosition(bigramsBuf, pos);
break;
}
if (isOffsetNegative(flags)) {
return origin - offset;
} else {
return origin + offset;
}
}
} // namespace latinime

88
native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_reading_utils.h Normal file
View File

@ -0,0 +1,88 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef LATINIME_BIGRAM_LIST_READING_UTILS_H
#define LATINIME_BIGRAM_LIST_READING_UTILS_H
#include <stdint.h>
#include "defines.h"
#include "suggest/core/dictionary/byte_array_utils.h"
namespace latinime {
class BigramListReadingUtils {
public:
typedef uint8_t BigramFlags;
static AK_FORCE_INLINE BigramFlags getFlagsAndForwardPointer(
const uint8_t *const bigramsBuf, int *const pos) {
return ByteArrayUtils::readUint8AndAdvancePosition(bigramsBuf, pos);
}
static AK_FORCE_INLINE int getProbabilityFromFlags(const BigramFlags flags) {
return flags & MASK_ATTRIBUTE_PROBABILITY;
}
static AK_FORCE_INLINE bool hasNext(const BigramFlags flags) {
return (flags & FLAG_ATTRIBUTE_HAS_NEXT) != 0;
}
// Bigrams reading methods
static AK_FORCE_INLINE void skipExistingBigrams(const uint8_t *const bigramsBuf,
int *const pos) {
BigramFlags flags = getFlagsAndForwardPointer(bigramsBuf, pos);
while (hasNext(flags)) {
*pos += attributeAddressSize(flags);
flags = getFlagsAndForwardPointer(bigramsBuf, pos);
}
*pos += attributeAddressSize(flags);
}
static int getBigramAddressAndForwardPointer(const uint8_t *const bigramsBuf,
const BigramFlags flags, int *const pos);
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(BigramListReadingUtils);
static const BigramFlags MASK_ATTRIBUTE_ADDRESS_TYPE;
static const BigramFlags FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE;
static const BigramFlags FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES;
static const BigramFlags FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES;
static const BigramFlags FLAG_ATTRIBUTE_OFFSET_NEGATIVE;
static const BigramFlags FLAG_ATTRIBUTE_HAS_NEXT;
static const BigramFlags MASK_ATTRIBUTE_PROBABILITY;
static const int ATTRIBUTE_ADDRESS_SHIFT;
static AK_FORCE_INLINE bool isOffsetNegative(const BigramFlags flags) {
return (flags & FLAG_ATTRIBUTE_OFFSET_NEGATIVE) != 0;
}
static AK_FORCE_INLINE int attributeAddressSize(const BigramFlags flags) {
return (flags & MASK_ATTRIBUTE_ADDRESS_TYPE) >> ATTRIBUTE_ADDRESS_SHIFT;
/* Note: this is a value-dependant optimization of what may probably be
more readably written this way:
switch (flags * BinaryFormat::MASK_ATTRIBUTE_ADDRESS_TYPE) {
case FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: return 1;
case FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: return 2;
case FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTE: return 3;
default: return 0;
}
*/
}
};
} // namespace latinime
#endif // LATINIME_BIGRAM_LIST_READING_UTILS_H

2
native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp
View File

@ -18,7 +18,7 @@
#include "suggest/core/dictionary/binary_dictionary_info.h"
#include "suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.h"
#include "suggest/policyimpl/dictionary/bigrams/bigram_list_policy.h"
#include "suggest/core/policy/dictionary_bigrams_structure_policy.h"
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h"
namespace latinime {

1
native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h
View File

@ -20,7 +20,6 @@
#include <stdint.h>
#include "defines.h"
#include "suggest/policyimpl/dictionary/bigrams/bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/patricia_trie_reading_utils.h"

2
native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h
View File

@ -21,7 +21,7 @@
#include "defines.h"
#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
#include "suggest/policyimpl/dictionary/bigrams/bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/bigram/bigram_list_policy.h"
namespace latinime {

2
native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h
View File

@ -21,7 +21,7 @@
#include "defines.h"
#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
#include "suggest/policyimpl/dictionary/bigrams/bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/bigram/bigram_list_policy.h"
namespace latinime {