nyancrimew/LatinIME
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge "Use extendable buffer for reading bigram and shortcut."

Browse source
This commit is contained in:
Keisuke Kuroyanagi 2013-08-27 08:58:45 +00:00 committed by Android (Google) Code Review
commit 8293a11d86
4 changed files with 180 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

80
native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h Normal file
View file

@ -0,0 +1,80 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef LATINIME_DYNAMIC_BIGRAM_LIST_POLICY_H
#define LATINIME_DYNAMIC_BIGRAM_LIST_POLICY_H
#include <stdint.h>
#include "defines.h"
#include "suggest/core/policy/dictionary_bigrams_structure_policy.h"
#include "suggest/policyimpl/dictionary/bigram/bigram_list_reading_utils.h"
#include "suggest/policyimpl/dictionary/utils/extendable_buffer.h"
namespace latinime {
/*
* This is a dynamic version of BigramListPolicy and supports an additional buffer.
*/
class DynamicBigramListPolicy : public DictionaryBigramsStructurePolicy {
public:
DynamicBigramListPolicy(const uint8_t *const bigramsBuf, const int bufSize,
const ExtendableBuffer *const additionalBuffer)
: mDictRoot(bigramsBuf), mBufSize(bufSize), mAdditionalBuffer(additionalBuffer) {}
~DynamicBigramListPolicy() {}
void getNextBigram(int *const outBigramPos, int *const outProbability, bool *const outHasNext,
int *const pos) const {
const bool usesAdditionalBuffer = *pos >= mBufSize;
const uint8_t *const buffer = (usesAdditionalBuffer) ?
mAdditionalBuffer->getBuffer() : mDictRoot;
if (usesAdditionalBuffer) {
*pos -= mBufSize;
}
const BigramListReadingUtils::BigramFlags flags =
BigramListReadingUtils::getFlagsAndForwardPointer(buffer, pos);
*outBigramPos = BigramListReadingUtils::getBigramAddressAndForwardPointer(
buffer, flags, pos);
if (usesAdditionalBuffer) {
*outBigramPos += mBufSize;
}
*outProbability = BigramListReadingUtils::getProbabilityFromFlags(flags);
*outHasNext = BigramListReadingUtils::hasNext(flags);
if (usesAdditionalBuffer) {
*pos += mBufSize;
}
}
void skipAllBigrams(int *const pos) const {
if (*pos >= mBufSize) {
*pos -= mBufSize;
BigramListReadingUtils::skipExistingBigrams(mAdditionalBuffer->getBuffer(), pos);
*pos += mBufSize;
} else {
BigramListReadingUtils::skipExistingBigrams(mDictRoot, pos);
}
}
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicBigramListPolicy);
const uint8_t *const mDictRoot;
const int mBufSize;
const ExtendableBuffer *const mAdditionalBuffer;
};
} // namespace latinime
#endif // LATINIME_DYNAMIC_BIGRAM_LIST_POLICY_H

94
native/jni/src/suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h Normal file
View file

@ -0,0 +1,94 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef LATINIME_DYNAMIC_SHORTCUT_LIST_POLICY_H
#define LATINIME_DYNAMIC_SHORTCUT_LIST_POLICY_H
#include <stdint.h>
#include "defines.h"
#include "suggest/core/policy/dictionary_shortcuts_structure_policy.h"
#include "suggest/policyimpl/dictionary/shortcut/shortcut_list_reading_utils.h"
#include "suggest/policyimpl/dictionary/utils/extendable_buffer.h"
namespace latinime {
/*
* This is a dynamic version of ShortcutListPolicy and supports an additional buffer.
*/
class DynamicShortcutListPolicy : public DictionaryShortcutsStructurePolicy {
public:
DynamicShortcutListPolicy(const uint8_t *const shortcutBuf, const int bufSize,
const ExtendableBuffer *const additionalBuffer)
: mShortcutsBuf(shortcutBuf), mBufSize(bufSize), mAdditionalBuffer(additionalBuffer) {}
~DynamicShortcutListPolicy() {}
int getStartPos(const int pos) const {
if (pos == NOT_A_DICT_POS) {
return NOT_A_DICT_POS;
}
return pos + ShortcutListReadingUtils::getShortcutListSizeFieldSize();
}
void getNextShortcut(const int maxCodePointCount, int *const outCodePoint,
int *const outCodePointCount, bool *const outIsWhitelist, bool *const outHasNext,
int *const pos) const {
const bool usesAdditionalBuffer = *pos >= mBufSize;
const uint8_t *const buffer = usesAdditionalBuffer
? mAdditionalBuffer->getBuffer() : mShortcutsBuf;
if (usesAdditionalBuffer) {
*pos -= mBufSize;
}
const ShortcutListReadingUtils::ShortcutFlags flags =
ShortcutListReadingUtils::getFlagsAndForwardPointer(buffer, pos);
if (outHasNext) {
*outHasNext = ShortcutListReadingUtils::hasNext(flags);
}
if (outIsWhitelist) {
*outIsWhitelist = ShortcutListReadingUtils::isWhitelist(flags);
}
if (outCodePoint) {
*outCodePointCount = ShortcutListReadingUtils::readShortcutTarget(
buffer, maxCodePointCount, outCodePoint, pos);
}
if (usesAdditionalBuffer) {
*pos += mBufSize;
}
}
void skipAllShortcuts(int *const pos) const {
if (*pos >= mBufSize) {
*pos -= mBufSize;
const int shortcutListSize = ShortcutListReadingUtils
::getShortcutListSizeAndForwardPointer(mAdditionalBuffer->getBuffer(), pos);
*pos += mBufSize + shortcutListSize;
} else {
const int shortcutListSize = ShortcutListReadingUtils
::getShortcutListSizeAndForwardPointer(mShortcutsBuf, pos);
*pos += shortcutListSize;
}
}
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicShortcutListPolicy);
const uint8_t *const mShortcutsBuf;
const int mBufSize;
const ExtendableBuffer *const mAdditionalBuffer;
};
} // namespace latinime
#endif // LATINIME_DYNAMIC_SHORTCUT_LIST_POLICY_H

4
native/jni/src/suggest/policyimpl/dictionary/shortcut/shortcut_list_reading_utils.h
View file

@ -50,6 +50,10 @@ class ShortcutListReadingUtils {
- SHORTCUT_LIST_SIZE_FIELD_SIZE;
}
static AK_FORCE_INLINE int getShortcutListSizeFieldSize() {
return SHORTCUT_LIST_SIZE_FIELD_SIZE;
}
static AK_FORCE_INLINE void skipShortcuts(const uint8_t *const dictRoot, int *const pos) {
const int shortcutListSize = getShortcutListSizeAndForwardPointer(dictRoot, pos);
*pos += shortcutListSize;

4
native/jni/src/suggest/policyimpl/dictionary/utils/extendable_buffer.h
View file

@ -30,12 +30,12 @@ class ExtendableBuffer {
public:
ExtendableBuffer() : mBuffer(INITIAL_BUFFER_SIZE), mUsedSize(0) {}
AK_FORCE_INLINE uint8_t *getBuffer() {
AK_FORCE_INLINE const uint8_t *getBuffer() const {
return &mBuffer[0];
}
// Return if the buffer is successfully extended or not.
AK_FORCE_INLINE bool extendBuffer() {
AK_FORCE_INLINE bool extendBuffer() {
if (mBuffer.size() + EXTEND_BUFFER_SIZE_STEP > MAX_BUFFER_SIZE) {
return false;
}