nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Make MmappedBuffer use byte array view. 

Bug: 16691311
Change-Id: I2122c01ee27c33e11dec52643925c069927bea2b
main
Keisuke Kuroyanagi 2014-08-01 19:26:01 +09:00
parent 0257e40345
commit c0c674cdc0
13 changed files with 76 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/single_dict_content.h
View File

@ -30,6 +30,7 @@
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
#include "utils/byte_array_view.h"
namespace latinime {
namespace backward {
@ -40,8 +41,9 @@ class SingleDictContent : public DictContent {
SingleDictContent(const char *const dictPath, const char *const contentFileName,
const bool isUpdatable)
: mMmappedBuffer(MmappedBuffer::openBuffer(dictPath, contentFileName, isUpdatable)),
mExpandableContentBuffer(mMmappedBuffer ? mMmappedBuffer->getBuffer() : nullptr,
mMmappedBuffer ? mMmappedBuffer->getBufferSize() : 0,
mExpandableContentBuffer(
mMmappedBuffer ? mMmappedBuffer->getReadWriteByteArrayView() :
ReadWriteByteArrayView(),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mIsValid(mMmappedBuffer) {}

14
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.h
View File

@ -31,6 +31,7 @@
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
#include "suggest/policyimpl/dictionary/utils/sparse_table.h"
#include "utils/byte_array_view.h"
namespace latinime {
namespace backward {
@ -50,15 +51,16 @@ class SparseTableDictContent : public DictContent {
mContentBuffer(
MmappedBuffer::openBuffer(dictPath, contentFileName, isUpdatable)),
mExpandableLookupTableBuffer(
mLookupTableBuffer ? mLookupTableBuffer->getBuffer() : nullptr,
mLookupTableBuffer ? mLookupTableBuffer->getBufferSize() : 0,
mLookupTableBuffer ? mLookupTableBuffer->getReadWriteByteArrayView() :
ReadWriteByteArrayView(),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mExpandableAddressTableBuffer(
mAddressTableBuffer ? mAddressTableBuffer->getBuffer() : nullptr,
mAddressTableBuffer ? mAddressTableBuffer->getBufferSize() : 0,
mAddressTableBuffer ? mAddressTableBuffer->getReadWriteByteArrayView() :
ReadWriteByteArrayView(),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mExpandableContentBuffer(mContentBuffer ? mContentBuffer->getBuffer() : nullptr,
mContentBuffer ? mContentBuffer->getBufferSize() : 0,
mExpandableContentBuffer(
mContentBuffer ? mContentBuffer->getReadWriteByteArrayView() :
ReadWriteByteArrayView(),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mAddressLookupTable(&mExpandableLookupTableBuffer, &mExpandableAddressTableBuffer,
sparseTableBlockSize, sparseTableDataSize),

11
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.cpp
View File

@ -30,6 +30,7 @@
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
#include "suggest/policyimpl/dictionary/utils/file_utils.h"
#include "utils/byte_array_view.h"
namespace latinime {
namespace backward {
@ -130,12 +131,12 @@ Ver4DictBuffers::Ver4DictBuffers(const char *const dictPath,
: mHeaderBuffer(std::move(headerBuffer)),
mDictBuffer(MmappedBuffer::openBuffer(dictPath,
Ver4DictConstants::TRIE_FILE_EXTENSION, isUpdatable)),
mHeaderPolicy(mHeaderBuffer->getBuffer(), formatVersion),
mExpandableHeaderBuffer(mHeaderBuffer ? mHeaderBuffer->getBuffer() : nullptr,
mHeaderPolicy.getSize(),
mHeaderPolicy(mHeaderBuffer->getReadOnlyByteArrayView().data(), formatVersion),
mExpandableHeaderBuffer(mHeaderBuffer->getReadWriteByteArrayView(),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mExpandableTrieBuffer(mDictBuffer ? mDictBuffer->getBuffer() : nullptr,
mDictBuffer ? mDictBuffer->getBufferSize() : 0,
mExpandableTrieBuffer(
mDictBuffer ? mDictBuffer->getReadWriteByteArrayView() :
ReadWriteByteArrayView(),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mTerminalPositionLookupTable(dictPath, isUpdatable),
mProbabilityDictContent(dictPath, mHeaderPolicy.hasHistoricalInfoOfWords(), isUpdatable),

8
native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp
View File

@ -31,6 +31,7 @@
#include "suggest/policyimpl/dictionary/utils/file_utils.h"
#include "suggest/policyimpl/dictionary/utils/format_utils.h"
#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
#include "utils/byte_array_view.h"
namespace latinime {
@ -110,7 +111,8 @@ template<class DictConstants, class DictBuffers, class DictBuffersPtr, class Str
return nullptr;
}
const FormatUtils::FORMAT_VERSION formatVersion = FormatUtils::detectFormatVersion(
mmappedBuffer->getBuffer(), mmappedBuffer->getBufferSize());
mmappedBuffer->getReadOnlyByteArrayView().data(),
mmappedBuffer->getReadOnlyByteArrayView().size());
switch (formatVersion) {
case FormatUtils::VERSION_2:
AKLOGE("Given path is a directory but the format is version 2. path: %s", path);
@ -172,8 +174,8 @@ template<class DictConstants, class DictBuffers, class DictBuffersPtr, class Str
if (!mmappedBuffer) {
return nullptr;
}
switch (FormatUtils::detectFormatVersion(mmappedBuffer->getBuffer(),
mmappedBuffer->getBufferSize())) {
switch (FormatUtils::detectFormatVersion(mmappedBuffer->getReadOnlyByteArrayView().data(),
mmappedBuffer->getReadOnlyByteArrayView().size())) {
case FormatUtils::VERSION_2:
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(
new PatriciaTriePolicy(std::move(mmappedBuffer)));

10
native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h
View File

@ -29,6 +29,7 @@
#include "suggest/policyimpl/dictionary/structure/v2/ver2_pt_node_array_reader.h"
#include "suggest/policyimpl/dictionary/utils/format_utils.h"
#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
#include "utils/byte_array_view.h"
namespace latinime {
@ -39,9 +40,12 @@ class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
public:
PatriciaTriePolicy(MmappedBuffer::MmappedBufferPtr mmappedBuffer)
: mMmappedBuffer(std::move(mmappedBuffer)),
mHeaderPolicy(mMmappedBuffer->getBuffer(), FormatUtils::VERSION_2),
mDictRoot(mMmappedBuffer->getBuffer() + mHeaderPolicy.getSize()),
mDictBufferSize(mMmappedBuffer->getBufferSize() - mHeaderPolicy.getSize()),
mHeaderPolicy(mMmappedBuffer->getReadOnlyByteArrayView().data(),
FormatUtils::VERSION_2),
mDictRoot(mMmappedBuffer->getReadOnlyByteArrayView().data()
+ mHeaderPolicy.getSize()),
mDictBufferSize(mMmappedBuffer->getReadOnlyByteArrayView().size()
- mHeaderPolicy.getSize()),
mBigramListPolicy(mDictRoot, mDictBufferSize), mShortcutListPolicy(mDictRoot),
mPtNodeReader(mDictRoot, mDictBufferSize, &mBigramListPolicy, &mShortcutListPolicy),
mPtNodeArrayReader(mDictRoot, mDictBufferSize),

3
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h
View File

@ -24,13 +24,14 @@
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
#include "utils/byte_array_view.h"
namespace latinime {
class SingleDictContent {
public:
SingleDictContent(uint8_t *const buffer, const int bufferSize)
: mExpandableContentBuffer(buffer, bufferSize,
: mExpandableContentBuffer(ReadWriteByteArrayView(buffer, bufferSize),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE) {}
SingleDictContent()

16
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h
View File

@ -24,6 +24,7 @@
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/sparse_table.h"
#include "utils/byte_array_view.h"
namespace latinime {
@ -32,14 +33,17 @@ class SparseTableDictContent {
public:
AK_FORCE_INLINE SparseTableDictContent(uint8_t *const *buffers, const int *bufferSizes,
const int sparseTableBlockSize, const int sparseTableDataSize)
: mExpandableLookupTableBuffer(buffers[LOOKUP_TABLE_BUFFER_INDEX],
bufferSizes[LOOKUP_TABLE_BUFFER_INDEX],
: mExpandableLookupTableBuffer(
ReadWriteByteArrayView(buffers[LOOKUP_TABLE_BUFFER_INDEX],
bufferSizes[LOOKUP_TABLE_BUFFER_INDEX]),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mExpandableAddressTableBuffer(buffers[ADDRESS_TABLE_BUFFER_INDEX],
bufferSizes[ADDRESS_TABLE_BUFFER_INDEX],
mExpandableAddressTableBuffer(
ReadWriteByteArrayView(buffers[ADDRESS_TABLE_BUFFER_INDEX],
bufferSizes[ADDRESS_TABLE_BUFFER_INDEX]),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mExpandableContentBuffer(buffers[CONTENT_BUFFER_INDEX],
bufferSizes[CONTENT_BUFFER_INDEX],
mExpandableContentBuffer(
ReadWriteByteArrayView(buffers[CONTENT_BUFFER_INDEX],
bufferSizes[CONTENT_BUFFER_INDEX]),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mAddressLookupTable(&mExpandableLookupTableBuffer, &mExpandableAddressTableBuffer,
sparseTableBlockSize, sparseTableDataSize) {}

25
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp
View File

@ -26,6 +26,7 @@
#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h"
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
#include "suggest/policyimpl/dictionary/utils/file_utils.h"
#include "utils/byte_array_view.h"
namespace latinime {
@ -46,14 +47,16 @@ namespace latinime {
}
std::vector<uint8_t *> buffers;
std::vector<int> bufferSizes;
uint8_t *const buffer = bodyBuffer->getBuffer();
const ReadWriteByteArrayView buffer = bodyBuffer->getReadWriteByteArrayView();
int position = 0;
while (position < bodyBuffer->getBufferSize()) {
const int bufferSize = ByteArrayUtils::readUint32AndAdvancePosition(buffer, &position);
buffers.push_back(buffer + position);
bufferSizes.push_back(bufferSize);
while (position < static_cast<int>(buffer.size())) {
const int bufferSize = ByteArrayUtils::readUint32AndAdvancePosition(
buffer.data(), &position);
const ReadWriteByteArrayView subBuffer = buffer.subView(position, bufferSize);
buffers.push_back(subBuffer.data());
bufferSizes.push_back(subBuffer.size());
position += bufferSize;
if (bufferSize < 0 || position < 0 || position > bodyBuffer->getBufferSize()) {
if (bufferSize < 0 || position < 0 || position > static_cast<int>(buffer.size())) {
AKLOGE("The dict body file is corrupted.");
return Ver4DictBuffersPtr(nullptr);
}
@ -177,12 +180,12 @@ Ver4DictBuffers::Ver4DictBuffers(MmappedBuffer::MmappedBufferPtr &&headerBuffer,
const FormatUtils::FORMAT_VERSION formatVersion,
const std::vector<uint8_t *> &contentBuffers, const std::vector<int> &contentBufferSizes)
: mHeaderBuffer(std::move(headerBuffer)), mDictBuffer(std::move(bodyBuffer)),
mHeaderPolicy(mHeaderBuffer->getBuffer(), formatVersion),
mExpandableHeaderBuffer(mHeaderBuffer ? mHeaderBuffer->getBuffer() : nullptr,
mHeaderPolicy.getSize(),
mHeaderPolicy(mHeaderBuffer->getReadOnlyByteArrayView().data(), formatVersion),
mExpandableHeaderBuffer(mHeaderBuffer->getReadWriteByteArrayView(),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mExpandableTrieBuffer(contentBuffers[Ver4DictConstants::TRIE_BUFFER_INDEX],
contentBufferSizes[Ver4DictConstants::TRIE_BUFFER_INDEX],
mExpandableTrieBuffer(
ReadWriteByteArrayView(contentBuffers[Ver4DictConstants::TRIE_BUFFER_INDEX],
contentBufferSizes[Ver4DictConstants::TRIE_BUFFER_INDEX]),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mTerminalPositionLookupTable(
contentBuffers[Ver4DictConstants::TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX],

7
native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h
View File

@ -35,15 +35,14 @@ class BufferWithExtendableBuffer {
public:
static const size_t DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE;
BufferWithExtendableBuffer(uint8_t *const originalBuffer, const int originalBufferSize,
BufferWithExtendableBuffer(const ReadWriteByteArrayView originalBuffer,
const int maxAdditionalBufferSize)
: mOriginalBuffer(originalBuffer, originalBufferSize),
mAdditionalBuffer(0), mUsedAdditionalBufferSize(0),
: mOriginalBuffer(originalBuffer), mAdditionalBuffer(), mUsedAdditionalBufferSize(0),
mMaxAdditionalBufferSize(maxAdditionalBufferSize) {}
// Without original buffer.
BufferWithExtendableBuffer(const int maxAdditionalBufferSize)
: mOriginalBuffer(), mAdditionalBuffer(0), mUsedAdditionalBufferSize(0),
: mOriginalBuffer(), mAdditionalBuffer(), mUsedAdditionalBufferSize(0),
mMaxAdditionalBufferSize(maxAdditionalBufferSize) {}
AK_FORCE_INLINE int getTailPosition() const {

16
native/jni/src/suggest/policyimpl/dictionary/utils/mmapped_buffer.h
View File

@ -21,6 +21,7 @@
#include <memory>
#include "defines.h"
#include "utils/byte_array_view.h"
namespace latinime {
@ -39,12 +40,12 @@ class MmappedBuffer {
~MmappedBuffer();
AK_FORCE_INLINE uint8_t *getBuffer() const {
return mBuffer;
ReadWriteByteArrayView getReadWriteByteArrayView() const {
return mByteArrayView;
}
AK_FORCE_INLINE int getBufferSize() const {
return mBufferSize;
ReadOnlyByteArrayView getReadOnlyByteArrayView() const {
return mByteArrayView.getReadOnlyView();
}
AK_FORCE_INLINE bool isUpdatable() const {
@ -55,18 +56,17 @@ class MmappedBuffer {
AK_FORCE_INLINE MmappedBuffer(uint8_t *const buffer, const int bufferSize,
void *const mmappedBuffer, const int alignedSize, const int mmapFd,
const bool isUpdatable)
: mBuffer(buffer), mBufferSize(bufferSize), mMmappedBuffer(mmappedBuffer),
: mByteArrayView(buffer, bufferSize), mMmappedBuffer(mmappedBuffer),
mAlignedSize(alignedSize), mMmapFd(mmapFd), mIsUpdatable(isUpdatable) {}
// Empty file. We have to handle an empty file as a valid part of a dictionary.
AK_FORCE_INLINE MmappedBuffer(const bool isUpdatable)
: mBuffer(nullptr), mBufferSize(0), mMmappedBuffer(nullptr), mAlignedSize(0),
: mByteArrayView(), mMmappedBuffer(nullptr), mAlignedSize(0),
mMmapFd(0), mIsUpdatable(isUpdatable) {}
DISALLOW_IMPLICIT_CONSTRUCTORS(MmappedBuffer);
uint8_t *const mBuffer;
const int mBufferSize;
const ReadWriteByteArrayView mByteArrayView;
void *const mMmappedBuffer;
const int mAlignedSize;
const int mMmapFd;

5
native/jni/src/suggest/policyimpl/dictionary/utils/trie_map.cpp
View File

@ -43,9 +43,8 @@ TrieMap::TrieMap() : mBuffer(MAX_BUFFER_SIZE) {
writeEntry(EMPTY_BITMAP_ENTRY, ROOT_BITMAP_ENTRY_INDEX);
}
TrieMap::TrieMap(uint8_t *const buffer, const int bufferSize)
: mBuffer(buffer, bufferSize,
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE) {}
TrieMap::TrieMap(const ReadWriteByteArrayView buffer)
: mBuffer(buffer, BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE) {}
void TrieMap::dump(const int from, const int to) const {
AKLOGI("BufSize: %d", mBuffer.getTailPosition());

3
native/jni/src/suggest/policyimpl/dictionary/utils/trie_map.h
View File

@ -24,6 +24,7 @@
#include "defines.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "utils/byte_array_view.h"
namespace latinime {
@ -161,7 +162,7 @@ class TrieMap {
TrieMap();
// Construct TrieMap using existing data in the memory region written by save().
TrieMap(uint8_t *const buffer, const int bufferSize);
TrieMap(const ReadWriteByteArrayView buffer);
void dump(const int from = 0, const int to = 0) const;
bool isNearSizeLimit() const {

5
native/jni/src/utils/byte_array_view.h
View File

@ -71,6 +71,11 @@ class ReadWriteByteArrayView {
return ReadOnlyByteArrayView(mPtr, mSize);
}
ReadWriteByteArrayView subView(const size_t start, const size_t n) const {
ASSERT(start + n <= mSize);
return ReadWriteByteArrayView(mPtr + start, n);
}
private:
DISALLOW_ASSIGNMENT_OPERATOR(ReadWriteByteArrayView);