diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/single_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/single_dict_content.h index 6433650b0..49f446814 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/single_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/single_dict_content.h @@ -30,6 +30,7 @@ #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h" #include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h" +#include "utils/byte_array_view.h" namespace latinime { namespace backward { @@ -40,8 +41,9 @@ class SingleDictContent : public DictContent { SingleDictContent(const char *const dictPath, const char *const contentFileName, const bool isUpdatable) : mMmappedBuffer(MmappedBuffer::openBuffer(dictPath, contentFileName, isUpdatable)), - mExpandableContentBuffer(mMmappedBuffer ? mMmappedBuffer->getBuffer() : nullptr, - mMmappedBuffer ? mMmappedBuffer->getBufferSize() : 0, + mExpandableContentBuffer( + mMmappedBuffer ? mMmappedBuffer->getReadWriteByteArrayView() : + ReadWriteByteArrayView(), BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), mIsValid(mMmappedBuffer) {} diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.h index c7233edd3..3c626df11 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.h @@ -31,6 +31,7 @@ #include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h" #include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h" #include "suggest/policyimpl/dictionary/utils/sparse_table.h" +#include "utils/byte_array_view.h" namespace latinime { namespace backward { @@ -50,15 +51,16 @@ class SparseTableDictContent : public DictContent { mContentBuffer( MmappedBuffer::openBuffer(dictPath, contentFileName, isUpdatable)), mExpandableLookupTableBuffer( - mLookupTableBuffer ? mLookupTableBuffer->getBuffer() : nullptr, - mLookupTableBuffer ? mLookupTableBuffer->getBufferSize() : 0, + mLookupTableBuffer ? mLookupTableBuffer->getReadWriteByteArrayView() : + ReadWriteByteArrayView(), BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), mExpandableAddressTableBuffer( - mAddressTableBuffer ? mAddressTableBuffer->getBuffer() : nullptr, - mAddressTableBuffer ? mAddressTableBuffer->getBufferSize() : 0, + mAddressTableBuffer ? mAddressTableBuffer->getReadWriteByteArrayView() : + ReadWriteByteArrayView(), BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), - mExpandableContentBuffer(mContentBuffer ? mContentBuffer->getBuffer() : nullptr, - mContentBuffer ? mContentBuffer->getBufferSize() : 0, + mExpandableContentBuffer( + mContentBuffer ? mContentBuffer->getReadWriteByteArrayView() : + ReadWriteByteArrayView(), BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), mAddressLookupTable(&mExpandableLookupTableBuffer, &mExpandableAddressTableBuffer, sparseTableBlockSize, sparseTableDataSize), diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.cpp index 93f192976..3dfbd1c94 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.cpp @@ -30,6 +30,7 @@ #include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h" #include "suggest/policyimpl/dictionary/utils/file_utils.h" +#include "utils/byte_array_view.h" namespace latinime { namespace backward { @@ -130,12 +131,12 @@ Ver4DictBuffers::Ver4DictBuffers(const char *const dictPath, : mHeaderBuffer(std::move(headerBuffer)), mDictBuffer(MmappedBuffer::openBuffer(dictPath, Ver4DictConstants::TRIE_FILE_EXTENSION, isUpdatable)), - mHeaderPolicy(mHeaderBuffer->getBuffer(), formatVersion), - mExpandableHeaderBuffer(mHeaderBuffer ? mHeaderBuffer->getBuffer() : nullptr, - mHeaderPolicy.getSize(), + mHeaderPolicy(mHeaderBuffer->getReadOnlyByteArrayView().data(), formatVersion), + mExpandableHeaderBuffer(mHeaderBuffer->getReadWriteByteArrayView(), BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), - mExpandableTrieBuffer(mDictBuffer ? mDictBuffer->getBuffer() : nullptr, - mDictBuffer ? mDictBuffer->getBufferSize() : 0, + mExpandableTrieBuffer( + mDictBuffer ? mDictBuffer->getReadWriteByteArrayView() : + ReadWriteByteArrayView(), BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), mTerminalPositionLookupTable(dictPath, isUpdatable), mProbabilityDictContent(dictPath, mHeaderPolicy.hasHistoricalInfoOfWords(), isUpdatable), diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp index e4b5fa267..e4ea3da16 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp @@ -31,6 +31,7 @@ #include "suggest/policyimpl/dictionary/utils/file_utils.h" #include "suggest/policyimpl/dictionary/utils/format_utils.h" #include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h" +#include "utils/byte_array_view.h" namespace latinime { @@ -110,7 +111,8 @@ templategetBuffer(), mmappedBuffer->getBufferSize()); + mmappedBuffer->getReadOnlyByteArrayView().data(), + mmappedBuffer->getReadOnlyByteArrayView().size()); switch (formatVersion) { case FormatUtils::VERSION_2: AKLOGE("Given path is a directory but the format is version 2. path: %s", path); @@ -172,8 +174,8 @@ templategetBuffer(), - mmappedBuffer->getBufferSize())) { + switch (FormatUtils::detectFormatVersion(mmappedBuffer->getReadOnlyByteArrayView().data(), + mmappedBuffer->getReadOnlyByteArrayView().size())) { case FormatUtils::VERSION_2: return DictionaryStructureWithBufferPolicy::StructurePolicyPtr( new PatriciaTriePolicy(std::move(mmappedBuffer))); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h index 18a62a938..70351d147 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h @@ -29,6 +29,7 @@ #include "suggest/policyimpl/dictionary/structure/v2/ver2_pt_node_array_reader.h" #include "suggest/policyimpl/dictionary/utils/format_utils.h" #include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h" +#include "utils/byte_array_view.h" namespace latinime { @@ -39,9 +40,12 @@ class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { public: PatriciaTriePolicy(MmappedBuffer::MmappedBufferPtr mmappedBuffer) : mMmappedBuffer(std::move(mmappedBuffer)), - mHeaderPolicy(mMmappedBuffer->getBuffer(), FormatUtils::VERSION_2), - mDictRoot(mMmappedBuffer->getBuffer() + mHeaderPolicy.getSize()), - mDictBufferSize(mMmappedBuffer->getBufferSize() - mHeaderPolicy.getSize()), + mHeaderPolicy(mMmappedBuffer->getReadOnlyByteArrayView().data(), + FormatUtils::VERSION_2), + mDictRoot(mMmappedBuffer->getReadOnlyByteArrayView().data() + + mHeaderPolicy.getSize()), + mDictBufferSize(mMmappedBuffer->getReadOnlyByteArrayView().size() + - mHeaderPolicy.getSize()), mBigramListPolicy(mDictRoot, mDictBufferSize), mShortcutListPolicy(mDictRoot), mPtNodeReader(mDictRoot, mDictBufferSize, &mBigramListPolicy, &mShortcutListPolicy), mPtNodeArrayReader(mDictRoot, mDictBufferSize), diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h index 203c79095..921774181 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h @@ -24,13 +24,14 @@ #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h" +#include "utils/byte_array_view.h" namespace latinime { class SingleDictContent { public: SingleDictContent(uint8_t *const buffer, const int bufferSize) - : mExpandableContentBuffer(buffer, bufferSize, + : mExpandableContentBuffer(ReadWriteByteArrayView(buffer, bufferSize), BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE) {} SingleDictContent() diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h index d6659615e..c98dd11fd 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h @@ -24,6 +24,7 @@ #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/sparse_table.h" +#include "utils/byte_array_view.h" namespace latinime { @@ -32,14 +33,17 @@ class SparseTableDictContent { public: AK_FORCE_INLINE SparseTableDictContent(uint8_t *const *buffers, const int *bufferSizes, const int sparseTableBlockSize, const int sparseTableDataSize) - : mExpandableLookupTableBuffer(buffers[LOOKUP_TABLE_BUFFER_INDEX], - bufferSizes[LOOKUP_TABLE_BUFFER_INDEX], + : mExpandableLookupTableBuffer( + ReadWriteByteArrayView(buffers[LOOKUP_TABLE_BUFFER_INDEX], + bufferSizes[LOOKUP_TABLE_BUFFER_INDEX]), BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), - mExpandableAddressTableBuffer(buffers[ADDRESS_TABLE_BUFFER_INDEX], - bufferSizes[ADDRESS_TABLE_BUFFER_INDEX], + mExpandableAddressTableBuffer( + ReadWriteByteArrayView(buffers[ADDRESS_TABLE_BUFFER_INDEX], + bufferSizes[ADDRESS_TABLE_BUFFER_INDEX]), BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), - mExpandableContentBuffer(buffers[CONTENT_BUFFER_INDEX], - bufferSizes[CONTENT_BUFFER_INDEX], + mExpandableContentBuffer( + ReadWriteByteArrayView(buffers[CONTENT_BUFFER_INDEX], + bufferSizes[CONTENT_BUFFER_INDEX]), BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), mAddressLookupTable(&mExpandableLookupTableBuffer, &mExpandableAddressTableBuffer, sparseTableBlockSize, sparseTableDataSize) {} diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp index e3501081e..a2920e4b7 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp @@ -26,6 +26,7 @@ #include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" #include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h" #include "suggest/policyimpl/dictionary/utils/file_utils.h" +#include "utils/byte_array_view.h" namespace latinime { @@ -46,14 +47,16 @@ namespace latinime { } std::vector buffers; std::vector bufferSizes; - uint8_t *const buffer = bodyBuffer->getBuffer(); + const ReadWriteByteArrayView buffer = bodyBuffer->getReadWriteByteArrayView(); int position = 0; - while (position < bodyBuffer->getBufferSize()) { - const int bufferSize = ByteArrayUtils::readUint32AndAdvancePosition(buffer, &position); - buffers.push_back(buffer + position); - bufferSizes.push_back(bufferSize); + while (position < static_cast(buffer.size())) { + const int bufferSize = ByteArrayUtils::readUint32AndAdvancePosition( + buffer.data(), &position); + const ReadWriteByteArrayView subBuffer = buffer.subView(position, bufferSize); + buffers.push_back(subBuffer.data()); + bufferSizes.push_back(subBuffer.size()); position += bufferSize; - if (bufferSize < 0 || position < 0 || position > bodyBuffer->getBufferSize()) { + if (bufferSize < 0 || position < 0 || position > static_cast(buffer.size())) { AKLOGE("The dict body file is corrupted."); return Ver4DictBuffersPtr(nullptr); } @@ -177,12 +180,12 @@ Ver4DictBuffers::Ver4DictBuffers(MmappedBuffer::MmappedBufferPtr &&headerBuffer, const FormatUtils::FORMAT_VERSION formatVersion, const std::vector &contentBuffers, const std::vector &contentBufferSizes) : mHeaderBuffer(std::move(headerBuffer)), mDictBuffer(std::move(bodyBuffer)), - mHeaderPolicy(mHeaderBuffer->getBuffer(), formatVersion), - mExpandableHeaderBuffer(mHeaderBuffer ? mHeaderBuffer->getBuffer() : nullptr, - mHeaderPolicy.getSize(), + mHeaderPolicy(mHeaderBuffer->getReadOnlyByteArrayView().data(), formatVersion), + mExpandableHeaderBuffer(mHeaderBuffer->getReadWriteByteArrayView(), BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), - mExpandableTrieBuffer(contentBuffers[Ver4DictConstants::TRIE_BUFFER_INDEX], - contentBufferSizes[Ver4DictConstants::TRIE_BUFFER_INDEX], + mExpandableTrieBuffer( + ReadWriteByteArrayView(contentBuffers[Ver4DictConstants::TRIE_BUFFER_INDEX], + contentBufferSizes[Ver4DictConstants::TRIE_BUFFER_INDEX]), BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), mTerminalPositionLookupTable( contentBuffers[Ver4DictConstants::TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX], diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h index 24e2dd4c8..fad83aa25 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h @@ -35,15 +35,14 @@ class BufferWithExtendableBuffer { public: static const size_t DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE; - BufferWithExtendableBuffer(uint8_t *const originalBuffer, const int originalBufferSize, + BufferWithExtendableBuffer(const ReadWriteByteArrayView originalBuffer, const int maxAdditionalBufferSize) - : mOriginalBuffer(originalBuffer, originalBufferSize), - mAdditionalBuffer(0), mUsedAdditionalBufferSize(0), + : mOriginalBuffer(originalBuffer), mAdditionalBuffer(), mUsedAdditionalBufferSize(0), mMaxAdditionalBufferSize(maxAdditionalBufferSize) {} // Without original buffer. BufferWithExtendableBuffer(const int maxAdditionalBufferSize) - : mOriginalBuffer(), mAdditionalBuffer(0), mUsedAdditionalBufferSize(0), + : mOriginalBuffer(), mAdditionalBuffer(), mUsedAdditionalBufferSize(0), mMaxAdditionalBufferSize(maxAdditionalBufferSize) {} AK_FORCE_INLINE int getTailPosition() const { diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/mmapped_buffer.h b/native/jni/src/suggest/policyimpl/dictionary/utils/mmapped_buffer.h index 8460087ab..e25310373 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/mmapped_buffer.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/mmapped_buffer.h @@ -21,6 +21,7 @@ #include #include "defines.h" +#include "utils/byte_array_view.h" namespace latinime { @@ -39,12 +40,12 @@ class MmappedBuffer { ~MmappedBuffer(); - AK_FORCE_INLINE uint8_t *getBuffer() const { - return mBuffer; + ReadWriteByteArrayView getReadWriteByteArrayView() const { + return mByteArrayView; } - AK_FORCE_INLINE int getBufferSize() const { - return mBufferSize; + ReadOnlyByteArrayView getReadOnlyByteArrayView() const { + return mByteArrayView.getReadOnlyView(); } AK_FORCE_INLINE bool isUpdatable() const { @@ -55,18 +56,17 @@ class MmappedBuffer { AK_FORCE_INLINE MmappedBuffer(uint8_t *const buffer, const int bufferSize, void *const mmappedBuffer, const int alignedSize, const int mmapFd, const bool isUpdatable) - : mBuffer(buffer), mBufferSize(bufferSize), mMmappedBuffer(mmappedBuffer), + : mByteArrayView(buffer, bufferSize), mMmappedBuffer(mmappedBuffer), mAlignedSize(alignedSize), mMmapFd(mmapFd), mIsUpdatable(isUpdatable) {} // Empty file. We have to handle an empty file as a valid part of a dictionary. AK_FORCE_INLINE MmappedBuffer(const bool isUpdatable) - : mBuffer(nullptr), mBufferSize(0), mMmappedBuffer(nullptr), mAlignedSize(0), + : mByteArrayView(), mMmappedBuffer(nullptr), mAlignedSize(0), mMmapFd(0), mIsUpdatable(isUpdatable) {} DISALLOW_IMPLICIT_CONSTRUCTORS(MmappedBuffer); - uint8_t *const mBuffer; - const int mBufferSize; + const ReadWriteByteArrayView mByteArrayView; void *const mMmappedBuffer; const int mAlignedSize; const int mMmapFd; diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/trie_map.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/trie_map.cpp index 2904b1e77..407b8efd0 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/trie_map.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/trie_map.cpp @@ -43,9 +43,8 @@ TrieMap::TrieMap() : mBuffer(MAX_BUFFER_SIZE) { writeEntry(EMPTY_BITMAP_ENTRY, ROOT_BITMAP_ENTRY_INDEX); } -TrieMap::TrieMap(uint8_t *const buffer, const int bufferSize) - : mBuffer(buffer, bufferSize, - BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE) {} +TrieMap::TrieMap(const ReadWriteByteArrayView buffer) + : mBuffer(buffer, BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE) {} void TrieMap::dump(const int from, const int to) const { AKLOGI("BufSize: %d", mBuffer.getTailPosition()); diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/trie_map.h b/native/jni/src/suggest/policyimpl/dictionary/utils/trie_map.h index 8b33346e6..a294ab876 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/trie_map.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/trie_map.h @@ -24,6 +24,7 @@ #include "defines.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" +#include "utils/byte_array_view.h" namespace latinime { @@ -161,7 +162,7 @@ class TrieMap { TrieMap(); // Construct TrieMap using existing data in the memory region written by save(). - TrieMap(uint8_t *const buffer, const int bufferSize); + TrieMap(const ReadWriteByteArrayView buffer); void dump(const int from = 0, const int to = 0) const; bool isNearSizeLimit() const { diff --git a/native/jni/src/utils/byte_array_view.h b/native/jni/src/utils/byte_array_view.h index d13999c16..2c97c6d58 100644 --- a/native/jni/src/utils/byte_array_view.h +++ b/native/jni/src/utils/byte_array_view.h @@ -71,6 +71,11 @@ class ReadWriteByteArrayView { return ReadOnlyByteArrayView(mPtr, mSize); } + ReadWriteByteArrayView subView(const size_t start, const size_t n) const { + ASSERT(start + n <= mSize); + return ReadWriteByteArrayView(mPtr + start, n); + } + private: DISALLOW_ASSIGNMENT_OPERATOR(ReadWriteByteArrayView);