nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

am 155fa8c6: am c0c674cd: Make MmappedBuffer use byte array view. 

* commit '155fa8c6f7030001906da4b6a97e4d6d26f96c6b':
  Make MmappedBuffer use byte array view.
main
Keisuke Kuroyanagi 2014-08-01 21:00:20 +00:00 committed by Android Git Automerger
parent 982c4bc5d3 155fa8c6f7
commit cac2ff7495
13 changed files with 76 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/single_dict_content.h
View File

@ -30,6 +30,7 @@
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
#include "utils/byte_array_view.h"
namespace latinime {
namespace backward {
@ -40,8 +41,9 @@ class SingleDictContent : public DictContent {
SingleDictContent(const char *const dictPath, const char *const contentFileName,
const bool isUpdatable)
: mMmappedBuffer(MmappedBuffer::openBuffer(dictPath, contentFileName, isUpdatable)),
mExpandableContentBuffer(mMmappedBuffer ? mMmappedBuffer->getBuffer() : nullptr,
mMmappedBuffer ? mMmappedBuffer->getBufferSize() : 0,
mExpandableContentBuffer(
mMmappedBuffer ? mMmappedBuffer->getReadWriteByteArrayView() :
ReadWriteByteArrayView(),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mIsValid(mMmappedBuffer) {}

14
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.h
View File

@ -31,6 +31,7 @@
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
#include "suggest/policyimpl/dictionary/utils/sparse_table.h"
#include "utils/byte_array_view.h"
namespace latinime {
namespace backward {
@ -50,15 +51,16 @@ class SparseTableDictContent : public DictContent {
mContentBuffer(
MmappedBuffer::openBuffer(dictPath, contentFileName, isUpdatable)),
mExpandableLookupTableBuffer(
mLookupTableBuffer ? mLookupTableBuffer->getBuffer() : nullptr,
mLookupTableBuffer ? mLookupTableBuffer->getBufferSize() : 0,
mLookupTableBuffer ? mLookupTableBuffer->getReadWriteByteArrayView() :
ReadWriteByteArrayView(),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mExpandableAddressTableBuffer(
mAddressTableBuffer ? mAddressTableBuffer->getBuffer() : nullptr,
mAddressTableBuffer ? mAddressTableBuffer->getBufferSize() : 0,
mAddressTableBuffer ? mAddressTableBuffer->getReadWriteByteArrayView() :
ReadWriteByteArrayView(),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mExpandableContentBuffer(mContentBuffer ? mContentBuffer->getBuffer() : nullptr,
mContentBuffer ? mContentBuffer->getBufferSize() : 0,
mExpandableContentBuffer(
mContentBuffer ? mContentBuffer->getReadWriteByteArrayView() :
ReadWriteByteArrayView(),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mAddressLookupTable(&mExpandableLookupTableBuffer, &mExpandableAddressTableBuffer,
sparseTableBlockSize, sparseTableDataSize),

11
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.cpp
View File

@ -30,6 +30,7 @@
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
#include "suggest/policyimpl/dictionary/utils/file_utils.h"
#include "utils/byte_array_view.h"
namespace latinime {
namespace backward {
@ -130,12 +131,12 @@ Ver4DictBuffers::Ver4DictBuffers(const char *const dictPath,
: mHeaderBuffer(std::move(headerBuffer)),
mDictBuffer(MmappedBuffer::openBuffer(dictPath,
Ver4DictConstants::TRIE_FILE_EXTENSION, isUpdatable)),
mHeaderPolicy(mHeaderBuffer->getBuffer(), formatVersion),
mExpandableHeaderBuffer(mHeaderBuffer ? mHeaderBuffer->getBuffer() : nullptr,
mHeaderPolicy.getSize(),
mHeaderPolicy(mHeaderBuffer->getReadOnlyByteArrayView().data(), formatVersion),
mExpandableHeaderBuffer(mHeaderBuffer->getReadWriteByteArrayView(),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mExpandableTrieBuffer(mDictBuffer ? mDictBuffer->getBuffer() : nullptr,
mDictBuffer ? mDictBuffer->getBufferSize() : 0,
mExpandableTrieBuffer(
mDictBuffer ? mDictBuffer->getReadWriteByteArrayView() :
ReadWriteByteArrayView(),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mTerminalPositionLookupTable(dictPath, isUpdatable),
mProbabilityDictContent(dictPath, mHeaderPolicy.hasHistoricalInfoOfWords(), isUpdatable),

8
native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp
View File

@ -31,6 +31,7 @@
#include "suggest/policyimpl/dictionary/utils/file_utils.h"
#include "suggest/policyimpl/dictionary/utils/format_utils.h"
#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
#include "utils/byte_array_view.h"
namespace latinime {
@ -110,7 +111,8 @@ template<class DictConstants, class DictBuffers, class DictBuffersPtr, class Str
return nullptr;
}
const FormatUtils::FORMAT_VERSION formatVersion = FormatUtils::detectFormatVersion(
mmappedBuffer->getBuffer(), mmappedBuffer->getBufferSize());
mmappedBuffer->getReadOnlyByteArrayView().data(),
mmappedBuffer->getReadOnlyByteArrayView().size());
switch (formatVersion) {
case FormatUtils::VERSION_2:
AKLOGE("Given path is a directory but the format is version 2. path: %s", path);
@ -172,8 +174,8 @@ template<class DictConstants, class DictBuffers, class DictBuffersPtr, class Str
if (!mmappedBuffer) {
return nullptr;
}
switch (FormatUtils::detectFormatVersion(mmappedBuffer->getBuffer(),
mmappedBuffer->getBufferSize())) {
switch (FormatUtils::detectFormatVersion(mmappedBuffer->getReadOnlyByteArrayView().data(),
mmappedBuffer->getReadOnlyByteArrayView().size())) {
case FormatUtils::VERSION_2:
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(
new PatriciaTriePolicy(std::move(mmappedBuffer)));

10
native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h
View File

@ -29,6 +29,7 @@
#include "suggest/policyimpl/dictionary/structure/v2/ver2_pt_node_array_reader.h"
#include "suggest/policyimpl/dictionary/utils/format_utils.h"
#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
#include "utils/byte_array_view.h"
namespace latinime {
@ -39,9 +40,12 @@ class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
public:
PatriciaTriePolicy(MmappedBuffer::MmappedBufferPtr mmappedBuffer)
: mMmappedBuffer(std::move(mmappedBuffer)),
mHeaderPolicy(mMmappedBuffer->getBuffer(), FormatUtils::VERSION_2),
mDictRoot(mMmappedBuffer->getBuffer() + mHeaderPolicy.getSize()),
mDictBufferSize(mMmappedBuffer->getBufferSize() - mHeaderPolicy.getSize()),
mHeaderPolicy(mMmappedBuffer->getReadOnlyByteArrayView().data(),
FormatUtils::VERSION_2),
mDictRoot(mMmappedBuffer->getReadOnlyByteArrayView().data()
+ mHeaderPolicy.getSize()),
mDictBufferSize(mMmappedBuffer->getReadOnlyByteArrayView().size()
- mHeaderPolicy.getSize()),
mBigramListPolicy(mDictRoot, mDictBufferSize), mShortcutListPolicy(mDictRoot),
mPtNodeReader(mDictRoot, mDictBufferSize, &mBigramListPolicy, &mShortcutListPolicy),
mPtNodeArrayReader(mDictRoot, mDictBufferSize),

3
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h
View File

@ -24,13 +24,14 @@
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
#include "utils/byte_array_view.h"
namespace latinime {
class SingleDictContent {
public:
SingleDictContent(uint8_t *const buffer, const int bufferSize)
: mExpandableContentBuffer(buffer, bufferSize,
: mExpandableContentBuffer(ReadWriteByteArrayView(buffer, bufferSize),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE) {}
SingleDictContent()

16
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h
View File

@ -24,6 +24,7 @@
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/sparse_table.h"
#include "utils/byte_array_view.h"
namespace latinime {
@ -32,14 +33,17 @@ class SparseTableDictContent {
public:
AK_FORCE_INLINE SparseTableDictContent(uint8_t *const *buffers, const int *bufferSizes,
const int sparseTableBlockSize, const int sparseTableDataSize)
: mExpandableLookupTableBuffer(buffers[LOOKUP_TABLE_BUFFER_INDEX],
bufferSizes[LOOKUP_TABLE_BUFFER_INDEX],
: mExpandableLookupTableBuffer(
ReadWriteByteArrayView(buffers[LOOKUP_TABLE_BUFFER_INDEX],
bufferSizes[LOOKUP_TABLE_BUFFER_INDEX]),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mExpandableAddressTableBuffer(buffers[ADDRESS_TABLE_BUFFER_INDEX],
bufferSizes[ADDRESS_TABLE_BUFFER_INDEX],
mExpandableAddressTableBuffer(
ReadWriteByteArrayView(buffers[ADDRESS_TABLE_BUFFER_INDEX],
bufferSizes[ADDRESS_TABLE_BUFFER_INDEX]),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mExpandableContentBuffer(buffers[CONTENT_BUFFER_INDEX],
bufferSizes[CONTENT_BUFFER_INDEX],
mExpandableContentBuffer(
ReadWriteByteArrayView(buffers[CONTENT_BUFFER_INDEX],
bufferSizes[CONTENT_BUFFER_INDEX]),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mAddressLookupTable(&mExpandableLookupTableBuffer, &mExpandableAddressTableBuffer,
sparseTableBlockSize, sparseTableDataSize) {}

25
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp
View File

@ -26,6 +26,7 @@
#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h"
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
#include "suggest/policyimpl/dictionary/utils/file_utils.h"
#include "utils/byte_array_view.h"
namespace latinime {
@ -46,14 +47,16 @@ namespace latinime {
}
std::vector<uint8_t *> buffers;
std::vector<int> bufferSizes;
uint8_t *const buffer = bodyBuffer->getBuffer();
const ReadWriteByteArrayView buffer = bodyBuffer->getReadWriteByteArrayView();
int position = 0;
while (position < bodyBuffer->getBufferSize()) {
const int bufferSize = ByteArrayUtils::readUint32AndAdvancePosition(buffer, &position);
buffers.push_back(buffer + position);
bufferSizes.push_back(bufferSize);
while (position < static_cast<int>(buffer.size())) {
const int bufferSize = ByteArrayUtils::readUint32AndAdvancePosition(
buffer.data(), &position);
const ReadWriteByteArrayView subBuffer = buffer.subView(position, bufferSize);
buffers.push_back(subBuffer.data());
bufferSizes.push_back(subBuffer.size());
position += bufferSize;
if (bufferSize < 0 || position < 0 || position > bodyBuffer->getBufferSize()) {
if (bufferSize < 0 || position < 0 || position > static_cast<int>(buffer.size())) {
AKLOGE("The dict body file is corrupted.");
return Ver4DictBuffersPtr(nullptr);
}
@ -177,12 +180,12 @@ Ver4DictBuffers::Ver4DictBuffers(MmappedBuffer::MmappedBufferPtr &&headerBuffer,
const FormatUtils::FORMAT_VERSION formatVersion,
const std::vector<uint8_t *> &contentBuffers, const std::vector<int> &contentBufferSizes)
: mHeaderBuffer(std::move(headerBuffer)), mDictBuffer(std::move(bodyBuffer)),
mHeaderPolicy(mHeaderBuffer->getBuffer(), formatVersion),
mExpandableHeaderBuffer(mHeaderBuffer ? mHeaderBuffer->getBuffer() : nullptr,
mHeaderPolicy.getSize(),
mHeaderPolicy(mHeaderBuffer->getReadOnlyByteArrayView().data(), formatVersion),
mExpandableHeaderBuffer(mHeaderBuffer->getReadWriteByteArrayView(),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mExpandableTrieBuffer(contentBuffers[Ver4DictConstants::TRIE_BUFFER_INDEX],
contentBufferSizes[Ver4DictConstants::TRIE_BUFFER_INDEX],
mExpandableTrieBuffer(
ReadWriteByteArrayView(contentBuffers[Ver4DictConstants::TRIE_BUFFER_INDEX],
contentBufferSizes[Ver4DictConstants::TRIE_BUFFER_INDEX]),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mTerminalPositionLookupTable(
contentBuffers[Ver4DictConstants::TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX],

7
native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h
View File

@ -35,15 +35,14 @@ class BufferWithExtendableBuffer {
public:
static const size_t DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE;
BufferWithExtendableBuffer(uint8_t *const originalBuffer, const int originalBufferSize,
BufferWithExtendableBuffer(const ReadWriteByteArrayView originalBuffer,
const int maxAdditionalBufferSize)
: mOriginalBuffer(originalBuffer, originalBufferSize),
mAdditionalBuffer(0), mUsedAdditionalBufferSize(0),
: mOriginalBuffer(originalBuffer), mAdditionalBuffer(), mUsedAdditionalBufferSize(0),
mMaxAdditionalBufferSize(maxAdditionalBufferSize) {}
// Without original buffer.
BufferWithExtendableBuffer(const int maxAdditionalBufferSize)
: mOriginalBuffer(), mAdditionalBuffer(0), mUsedAdditionalBufferSize(0),
: mOriginalBuffer(), mAdditionalBuffer(), mUsedAdditionalBufferSize(0),
mMaxAdditionalBufferSize(maxAdditionalBufferSize) {}
AK_FORCE_INLINE int getTailPosition() const {

16
native/jni/src/suggest/policyimpl/dictionary/utils/mmapped_buffer.h
View File

@ -21,6 +21,7 @@
#include <memory>
#include "defines.h"
#include "utils/byte_array_view.h"
namespace latinime {
@ -39,12 +40,12 @@ class MmappedBuffer {
~MmappedBuffer();
AK_FORCE_INLINE uint8_t *getBuffer() const {
return mBuffer;
ReadWriteByteArrayView getReadWriteByteArrayView() const {
return mByteArrayView;
}
AK_FORCE_INLINE int getBufferSize() const {
return mBufferSize;
ReadOnlyByteArrayView getReadOnlyByteArrayView() const {
return mByteArrayView.getReadOnlyView();
}
AK_FORCE_INLINE bool isUpdatable() const {
@ -55,18 +56,17 @@ class MmappedBuffer {
AK_FORCE_INLINE MmappedBuffer(uint8_t *const buffer, const int bufferSize,
void *const mmappedBuffer, const int alignedSize, const int mmapFd,
const bool isUpdatable)
: mBuffer(buffer), mBufferSize(bufferSize), mMmappedBuffer(mmappedBuffer),
: mByteArrayView(buffer, bufferSize), mMmappedBuffer(mmappedBuffer),
mAlignedSize(alignedSize), mMmapFd(mmapFd), mIsUpdatable(isUpdatable) {}
// Empty file. We have to handle an empty file as a valid part of a dictionary.
AK_FORCE_INLINE MmappedBuffer(const bool isUpdatable)
: mBuffer(nullptr), mBufferSize(0), mMmappedBuffer(nullptr), mAlignedSize(0),
: mByteArrayView(), mMmappedBuffer(nullptr), mAlignedSize(0),
mMmapFd(0), mIsUpdatable(isUpdatable) {}
DISALLOW_IMPLICIT_CONSTRUCTORS(MmappedBuffer);
uint8_t *const mBuffer;
const int mBufferSize;
const ReadWriteByteArrayView mByteArrayView;
void *const mMmappedBuffer;
const int mAlignedSize;
const int mMmapFd;

5
native/jni/src/suggest/policyimpl/dictionary/utils/trie_map.cpp
View File

@ -43,9 +43,8 @@ TrieMap::TrieMap() : mBuffer(MAX_BUFFER_SIZE) {
writeEntry(EMPTY_BITMAP_ENTRY, ROOT_BITMAP_ENTRY_INDEX);
}
TrieMap::TrieMap(uint8_t *const buffer, const int bufferSize)
: mBuffer(buffer, bufferSize,
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE) {}
TrieMap::TrieMap(const ReadWriteByteArrayView buffer)
: mBuffer(buffer, BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE) {}
void TrieMap::dump(const int from, const int to) const {
AKLOGI("BufSize: %d", mBuffer.getTailPosition());

3
native/jni/src/suggest/policyimpl/dictionary/utils/trie_map.h
View File

@ -24,6 +24,7 @@
#include "defines.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "utils/byte_array_view.h"
namespace latinime {
@ -161,7 +162,7 @@ class TrieMap {
TrieMap();
// Construct TrieMap using existing data in the memory region written by save().
TrieMap(uint8_t *const buffer, const int bufferSize);
TrieMap(const ReadWriteByteArrayView buffer);
void dump(const int from = 0, const int to = 0) const;
bool isNearSizeLimit() const {

5
native/jni/src/utils/byte_array_view.h
View File

@ -71,6 +71,11 @@ class ReadWriteByteArrayView {
return ReadOnlyByteArrayView(mPtr, mSize);
}
ReadWriteByteArrayView subView(const size_t start, const size_t n) const {
ASSERT(start + n <= mSize);
return ReadWriteByteArrayView(mPtr + start, n);
}
private:
DISALLOW_ASSIGNMENT_OPERATOR(ReadWriteByteArrayView);