nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge "Implement SparseTable in native code."

main
Keisuke Kuroyanagi 2013-11-01 19:00:01 +00:00 committed by Android (Google) Code Review
parent 8eb6f961f1 1c62341de6
commit 2d3374bb58
10 changed files with 134 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
native/jni/Android.mk
View File

@ -96,7 +96,8 @@ LATIN_IME_CORE_SRC_FILES := \
dict_file_writing_utils.cpp \
forgetting_curve_utils.cpp \
format_utils.cpp \
mmapped_buffer.cpp) \
mmapped_buffer.cpp \
sparse_table.cpp) \
suggest/policyimpl/gesture/gesture_suggest_policy_factory.cpp \
$(addprefix suggest/policyimpl/typing/, \
scoring_params.cpp \

9
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h
View File

@ -21,6 +21,7 @@
#include "suggest/policyimpl/dictionary/structure/v4/content/dict_content.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
#include "suggest/policyimpl/dictionary/utils/sparse_table.h"
namespace latinime {
@ -29,7 +30,8 @@ class SparseTableDictContent : public DictContent {
public:
AK_FORCE_INLINE SparseTableDictContent(const char *const dictDirPath,
const char *const lookupTableFileName, const char *const addressTableFileName,
const char *const contentFileName, const bool isUpdatable)
const char *const contentFileName, const bool isUpdatable,
const int sparseTableBlockSize, const int sparseTableDataSize)
: mLookupTableBuffer(
MmappedBuffer::openBuffer(dictDirPath, lookupTableFileName, isUpdatable)),
mAddressTableBuffer(
@ -45,7 +47,9 @@ class SparseTableDictContent : public DictContent {
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mExpandableContentBuffer(mContentBuffer.get() ? mContentBuffer.get()->getBuffer() : 0,
mContentBuffer.get() ? mContentBuffer.get()->getBufferSize() : 0,
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE) {}
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mAddressLookupTable(&mExpandableLookupTableBuffer, &mExpandableAddressTableBuffer,
sparseTableBlockSize, sparseTableDataSize) {}
virtual ~SparseTableDictContent() {}
@ -64,6 +68,7 @@ class SparseTableDictContent : public DictContent {
BufferWithExtendableBuffer mExpandableLookupTableBuffer;
BufferWithExtendableBuffer mExpandableAddressTableBuffer;
BufferWithExtendableBuffer mExpandableContentBuffer;
SparseTable mAddressLookupTable;
};
} // namespace latinime
#endif /* LATINIME_SPARSE_TABLE_DICT_CONTENT_H */

8
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h
View File

@ -67,11 +67,15 @@ class Ver4DictBuffers {
mBigramDictContent(dictDirPath,
Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION,
Ver4DictConstants::BIGRAM_CONTENT_TABLE_FILE_EXTENSION,
Ver4DictConstants::BIGRAM_FILE_EXTENSION, isUpdatable),
Ver4DictConstants::BIGRAM_FILE_EXTENSION, isUpdatable,
Ver4DictConstants::BIGRAM_ADDRESS_TABLE_BLOCK_SIZE,
Ver4DictConstants::BIGRAM_ADDRESS_TABLE_DATA_SIZE),
mShortcutDictContent(dictDirPath,
Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION,
Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION,
Ver4DictConstants::SHORTCUT_FILE_EXTENSION, isUpdatable) {}
Ver4DictConstants::SHORTCUT_FILE_EXTENSION, isUpdatable,
Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE,
Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_DATA_SIZE) {}
const MmappedBuffer::MmappedBufferPtr mDictBuffer;
SingleDictContent mTerminalAddressTable;

5
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp
View File

@ -34,4 +34,9 @@ const int Ver4DictConstants::NOT_A_TERMINAL_ID = -1;
const int Ver4DictConstants::PROBABILITY_SIZE = 1;
const int Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE = 1;
const int Ver4DictConstants::BIGRAM_ADDRESS_TABLE_BLOCK_SIZE = 4;
const int Ver4DictConstants::BIGRAM_ADDRESS_TABLE_DATA_SIZE = 4;
const int Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE = 16;
const int Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_DATA_SIZE = 4;
} // namespace latinime

4
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h
View File

@ -38,6 +38,10 @@ class Ver4DictConstants {
static const int PROBABILITY_SIZE;
static const int FLAGS_IN_PROBABILITY_FILE_SIZE;
static const int BIGRAM_ADDRESS_TABLE_BLOCK_SIZE;
static const int BIGRAM_ADDRESS_TABLE_DATA_SIZE;
static const int SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE;
static const int SHORTCUT_ADDRESS_TABLE_DATA_SIZE;
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4DictConstants);
};

17
native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.cpp
View File

@ -23,17 +23,16 @@ const int BufferWithExtendableBuffer::NEAR_BUFFER_LIMIT_THRESHOLD_PERCENTILE = 9
// TODO: Needs to allocate larger memory corresponding to the current vector size.
const size_t BufferWithExtendableBuffer::EXTEND_ADDITIONAL_BUFFER_SIZE_STEP = 128 * 1024;
uint32_t BufferWithExtendableBuffer::readUint(const int size, const int pos) const {
const bool readingPosIsInAdditionalBuffer = isInAdditionalBuffer(pos);
const int posInBuffer = readingPosIsInAdditionalBuffer ? pos - mOriginalBufferSize : pos;
return ByteArrayUtils::readUint(getBuffer(readingPosIsInAdditionalBuffer), size, posInBuffer);
}
uint32_t BufferWithExtendableBuffer::readUintAndAdvancePosition(const int size,
int *const pos) const {
const bool readingPosIsInAdditionalBuffer = isInAdditionalBuffer(*pos);
if (readingPosIsInAdditionalBuffer) {
*pos -= mOriginalBufferSize;
}
const int value = ByteArrayUtils::readUintAndAdvancePosition(
getBuffer(readingPosIsInAdditionalBuffer), size, pos);
if (readingPosIsInAdditionalBuffer) {
*pos += mOriginalBufferSize;
}
const int value = readUint(size, *pos);
*pos += size;
return value;
}

2
native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h
View File

@ -71,6 +71,8 @@ class BufferWithExtendableBuffer {
}
}
uint32_t readUint(const int size, const int pos) const;
uint32_t readUintAndAdvancePosition(const int size, int *const pos) const;
AK_FORCE_INLINE int getOriginalBufferSize() const {

12
native/jni/src/suggest/policyimpl/dictionary/utils/byte_array_utils.h
View File

@ -114,19 +114,19 @@ class ByteArrayUtils {
return buffer[(*pos)++];
}
static AK_FORCE_INLINE int readUintAndAdvancePosition(const uint8_t *const buffer,
const int size, int *const pos) {
static AK_FORCE_INLINE int readUint(const uint8_t *const buffer,
const int size, const int pos) {
// size must be in 1 to 4.
ASSERT(size >= 1 && size <= 4);
switch (size) {
case 1:
return ByteArrayUtils::readUint8AndAdvancePosition(buffer, pos);
return ByteArrayUtils::readUint8(buffer, pos);
case 2:
return ByteArrayUtils::readUint16AndAdvancePosition(buffer, pos);
return ByteArrayUtils::readUint16(buffer, pos);
case 3:
return ByteArrayUtils::readUint24AndAdvancePosition(buffer, pos);
return ByteArrayUtils::readUint24(buffer, pos);
case 4:
return ByteArrayUtils::readUint32AndAdvancePosition(buffer, pos);
return ByteArrayUtils::readUint32(buffer, pos);
default:
return 0;
}

42
native/jni/src/suggest/policyimpl/dictionary/utils/sparse_table.cpp Normal file
View File

@ -0,0 +1,42 @@
/*
* Copyright (C) 2013, The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "suggest/policyimpl/dictionary/utils/sparse_table.h"
namespace latinime {
const int SparseTable::NOT_EXIST = -1;
bool SparseTable::contains(const int id) const {
if (id < 0 || mIndexTableBuffer->getTailPosition() <= id * mDataSize) {
return false;
}
const int indexTableIndex = id / mBlockSize;
const int readingPos = indexTableIndex * mDataSize;
const int index = mIndexTableBuffer->readUint(mDataSize, readingPos);
return index != NOT_EXIST;
}
uint32_t SparseTable::get(const int id) const {
const int indexTableIndex = id / mBlockSize;
int readingPos = indexTableIndex * mDataSize;
const int index = mIndexTableBuffer->readUint(mDataSize, readingPos);
const int offset = id % mBlockSize;
readingPos = (index * mDataSize + offset) * mBlockSize;
return mContentTableBuffer->readUint(mDataSize, readingPos);
}
} // namespace latinime

52
native/jni/src/suggest/policyimpl/dictionary/utils/sparse_table.h Normal file
View File

@ -0,0 +1,52 @@
/*
* Copyright (C) 2013, The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef LATINIME_SPARSE_TABLE_H
#define LATINIME_SPARSE_TABLE_H
#include <stdint.h>
#include "defines.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
namespace latinime {
// Note that there is a corresponding implementation in SparseTable.java.
// TODO: Support multiple content buffers.
class SparseTable {
public:
SparseTable(BufferWithExtendableBuffer *const indexTableBuffer,
BufferWithExtendableBuffer *const contentTableBuffer, const int blockSize,
const int dataSize)
: mIndexTableBuffer(indexTableBuffer), mContentTableBuffer(contentTableBuffer),
mBlockSize(blockSize), mDataSize(dataSize) {}
bool contains(const int id) const;
uint32_t get(const int id) const;
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(SparseTable);
static const int NOT_EXIST;
BufferWithExtendableBuffer *const mIndexTableBuffer;
BufferWithExtendableBuffer *const mContentTableBuffer;
const int mBlockSize;
const int mDataSize;
};
} // namespace latinime
#endif /* LATINIME_SPARSE_TABLE_H */