am 155fa8c6: am c0c674cd: Make MmappedBuffer use byte array view.

* commit '155fa8c6f7030001906da4b6a97e4d6d26f96c6b':
  Make MmappedBuffer use byte array view.

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/single_dict_content.h

@@ -30,6 +30,7 @@
 #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
 #include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
 #include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
+#include "utils/byte_array_view.h"
 
 namespace latinime {
 namespace backward {
@@ -40,8 +41,9 @@ class SingleDictContent : public DictContent {
     SingleDictContent(const char *const dictPath, const char *const contentFileName,
             const bool isUpdatable)
             : mMmappedBuffer(MmappedBuffer::openBuffer(dictPath, contentFileName, isUpdatable)),
-              mExpandableContentBuffer(mMmappedBuffer ? mMmappedBuffer->getBuffer() : nullptr,
+              mExpandableContentBuffer(
-                      mMmappedBuffer ? mMmappedBuffer->getBufferSize() : 0,
+                      mMmappedBuffer ? mMmappedBuffer->getReadWriteByteArrayView() :
+                              ReadWriteByteArrayView(),
                       BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
               mIsValid(mMmappedBuffer) {}
 

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.h

@@ -31,6 +31,7 @@
 #include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
 #include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
 #include "suggest/policyimpl/dictionary/utils/sparse_table.h"
+#include "utils/byte_array_view.h"
 
 namespace latinime {
 namespace backward {
@@ -50,15 +51,16 @@ class SparseTableDictContent : public DictContent {
               mContentBuffer(
                       MmappedBuffer::openBuffer(dictPath, contentFileName, isUpdatable)),
               mExpandableLookupTableBuffer(
-                      mLookupTableBuffer ? mLookupTableBuffer->getBuffer() : nullptr,
+                      mLookupTableBuffer ? mLookupTableBuffer->getReadWriteByteArrayView() :
-                      mLookupTableBuffer ? mLookupTableBuffer->getBufferSize() : 0,
+                              ReadWriteByteArrayView(),
                       BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
               mExpandableAddressTableBuffer(
-                      mAddressTableBuffer ? mAddressTableBuffer->getBuffer() : nullptr,
+                      mAddressTableBuffer ? mAddressTableBuffer->getReadWriteByteArrayView() :
-                      mAddressTableBuffer ? mAddressTableBuffer->getBufferSize() : 0,
+                              ReadWriteByteArrayView(),
                       BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
-              mExpandableContentBuffer(mContentBuffer ? mContentBuffer->getBuffer() : nullptr,
+              mExpandableContentBuffer(
-                      mContentBuffer ? mContentBuffer->getBufferSize() : 0,
+                      mContentBuffer ? mContentBuffer->getReadWriteByteArrayView() :
+                              ReadWriteByteArrayView(),
                       BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
               mAddressLookupTable(&mExpandableLookupTableBuffer, &mExpandableAddressTableBuffer,
                       sparseTableBlockSize, sparseTableDataSize),

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.cpp

@@ -30,6 +30,7 @@
 
 #include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
 #include "suggest/policyimpl/dictionary/utils/file_utils.h"
+#include "utils/byte_array_view.h"
 
 namespace latinime {
 namespace backward {
@@ -130,12 +131,12 @@ Ver4DictBuffers::Ver4DictBuffers(const char *const dictPath,
         : mHeaderBuffer(std::move(headerBuffer)),
           mDictBuffer(MmappedBuffer::openBuffer(dictPath,
                   Ver4DictConstants::TRIE_FILE_EXTENSION, isUpdatable)),
-          mHeaderPolicy(mHeaderBuffer->getBuffer(), formatVersion),
+          mHeaderPolicy(mHeaderBuffer->getReadOnlyByteArrayView().data(), formatVersion),
-          mExpandableHeaderBuffer(mHeaderBuffer ? mHeaderBuffer->getBuffer() : nullptr,
+          mExpandableHeaderBuffer(mHeaderBuffer->getReadWriteByteArrayView(),
-                  mHeaderPolicy.getSize(),
                   BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
-          mExpandableTrieBuffer(mDictBuffer ? mDictBuffer->getBuffer() : nullptr,
+          mExpandableTrieBuffer(
-                  mDictBuffer ? mDictBuffer->getBufferSize() : 0,
+                  mDictBuffer ? mDictBuffer->getReadWriteByteArrayView() :
+                          ReadWriteByteArrayView(),
                   BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
           mTerminalPositionLookupTable(dictPath, isUpdatable),
           mProbabilityDictContent(dictPath, mHeaderPolicy.hasHistoricalInfoOfWords(), isUpdatable),

native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp

@@ -31,6 +31,7 @@
 #include "suggest/policyimpl/dictionary/utils/file_utils.h"
 #include "suggest/policyimpl/dictionary/utils/format_utils.h"
 #include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
+#include "utils/byte_array_view.h"
 
 namespace latinime {
 
@@ -110,7 +111,8 @@ template<class DictConstants, class DictBuffers, class DictBuffersPtr, class Str
         return nullptr;
     }
     const FormatUtils::FORMAT_VERSION formatVersion = FormatUtils::detectFormatVersion(
-            mmappedBuffer->getBuffer(), mmappedBuffer->getBufferSize());
+            mmappedBuffer->getReadOnlyByteArrayView().data(),
+            mmappedBuffer->getReadOnlyByteArrayView().size());
     switch (formatVersion) {
         case FormatUtils::VERSION_2:
             AKLOGE("Given path is a directory but the format is version 2. path: %s", path);
@@ -172,8 +174,8 @@ template<class DictConstants, class DictBuffers, class DictBuffersPtr, class Str
     if (!mmappedBuffer) {
         return nullptr;
     }
-    switch (FormatUtils::detectFormatVersion(mmappedBuffer->getBuffer(),
+    switch (FormatUtils::detectFormatVersion(mmappedBuffer->getReadOnlyByteArrayView().data(),
-            mmappedBuffer->getBufferSize())) {
+            mmappedBuffer->getReadOnlyByteArrayView().size())) {
         case FormatUtils::VERSION_2:
             return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(
                     new PatriciaTriePolicy(std::move(mmappedBuffer)));

native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h

@@ -29,6 +29,7 @@
 #include "suggest/policyimpl/dictionary/structure/v2/ver2_pt_node_array_reader.h"
 #include "suggest/policyimpl/dictionary/utils/format_utils.h"
 #include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
+#include "utils/byte_array_view.h"
 
 namespace latinime {
 
@@ -39,9 +40,12 @@ class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
  public:
     PatriciaTriePolicy(MmappedBuffer::MmappedBufferPtr mmappedBuffer)
             : mMmappedBuffer(std::move(mmappedBuffer)),
-              mHeaderPolicy(mMmappedBuffer->getBuffer(), FormatUtils::VERSION_2),
+              mHeaderPolicy(mMmappedBuffer->getReadOnlyByteArrayView().data(),
-              mDictRoot(mMmappedBuffer->getBuffer() + mHeaderPolicy.getSize()),
+                      FormatUtils::VERSION_2),
-              mDictBufferSize(mMmappedBuffer->getBufferSize() - mHeaderPolicy.getSize()),
+              mDictRoot(mMmappedBuffer->getReadOnlyByteArrayView().data()
+                      + mHeaderPolicy.getSize()),
+              mDictBufferSize(mMmappedBuffer->getReadOnlyByteArrayView().size()
+                      - mHeaderPolicy.getSize()),
               mBigramListPolicy(mDictRoot, mDictBufferSize), mShortcutListPolicy(mDictRoot),
               mPtNodeReader(mDictRoot, mDictBufferSize, &mBigramListPolicy, &mShortcutListPolicy),
               mPtNodeArrayReader(mDictRoot, mDictBufferSize),

native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h

@@ -24,13 +24,14 @@
 #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
 #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
 #include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
+#include "utils/byte_array_view.h"
 
 namespace latinime {
 
 class SingleDictContent {
  public:
     SingleDictContent(uint8_t *const buffer, const int bufferSize)
-            : mExpandableContentBuffer(buffer, bufferSize,
+            : mExpandableContentBuffer(ReadWriteByteArrayView(buffer, bufferSize),
                       BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE) {}
 
     SingleDictContent()

native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h

@@ -24,6 +24,7 @@
 #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
 #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
 #include "suggest/policyimpl/dictionary/utils/sparse_table.h"
+#include "utils/byte_array_view.h"
 
 namespace latinime {
 
@@ -32,14 +33,17 @@ class SparseTableDictContent {
  public:
     AK_FORCE_INLINE SparseTableDictContent(uint8_t *const *buffers, const int *bufferSizes,
             const int sparseTableBlockSize, const int sparseTableDataSize)
-            : mExpandableLookupTableBuffer(buffers[LOOKUP_TABLE_BUFFER_INDEX],
+            : mExpandableLookupTableBuffer(
-                      bufferSizes[LOOKUP_TABLE_BUFFER_INDEX],
+                      ReadWriteByteArrayView(buffers[LOOKUP_TABLE_BUFFER_INDEX],
+                              bufferSizes[LOOKUP_TABLE_BUFFER_INDEX]),
                       BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
-              mExpandableAddressTableBuffer(buffers[ADDRESS_TABLE_BUFFER_INDEX],
+              mExpandableAddressTableBuffer(
-                      bufferSizes[ADDRESS_TABLE_BUFFER_INDEX],
+                      ReadWriteByteArrayView(buffers[ADDRESS_TABLE_BUFFER_INDEX],
+                              bufferSizes[ADDRESS_TABLE_BUFFER_INDEX]),
                       BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
-              mExpandableContentBuffer(buffers[CONTENT_BUFFER_INDEX],
+              mExpandableContentBuffer(
-                      bufferSizes[CONTENT_BUFFER_INDEX],
+                      ReadWriteByteArrayView(buffers[CONTENT_BUFFER_INDEX],
+                              bufferSizes[CONTENT_BUFFER_INDEX]),
                       BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
               mAddressLookupTable(&mExpandableLookupTableBuffer, &mExpandableAddressTableBuffer,
                       sparseTableBlockSize, sparseTableDataSize) {}

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp

@@ -26,6 +26,7 @@
 #include "suggest/policyimpl/dictionary/utils/byte_array_utils.h"
 #include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
 #include "suggest/policyimpl/dictionary/utils/file_utils.h"
+#include "utils/byte_array_view.h"
 
 namespace latinime {
 
@@ -46,14 +47,16 @@ namespace latinime {
     }
     std::vector<uint8_t *> buffers;
     std::vector<int> bufferSizes;
-    uint8_t *const buffer = bodyBuffer->getBuffer();
+    const ReadWriteByteArrayView buffer = bodyBuffer->getReadWriteByteArrayView();
     int position = 0;
-    while (position < bodyBuffer->getBufferSize()) {
+    while (position < static_cast<int>(buffer.size())) {
-        const int bufferSize = ByteArrayUtils::readUint32AndAdvancePosition(buffer, &position);
+        const int bufferSize = ByteArrayUtils::readUint32AndAdvancePosition(
-        buffers.push_back(buffer + position);
+                buffer.data(), &position);
-        bufferSizes.push_back(bufferSize);
+        const ReadWriteByteArrayView subBuffer = buffer.subView(position, bufferSize);
+        buffers.push_back(subBuffer.data());
+        bufferSizes.push_back(subBuffer.size());
         position += bufferSize;
-        if (bufferSize < 0 || position < 0 || position > bodyBuffer->getBufferSize()) {
+        if (bufferSize < 0 || position < 0 || position > static_cast<int>(buffer.size())) {
             AKLOGE("The dict body file is corrupted.");
             return Ver4DictBuffersPtr(nullptr);
         }
@@ -177,12 +180,12 @@ Ver4DictBuffers::Ver4DictBuffers(MmappedBuffer::MmappedBufferPtr &&headerBuffer,
         const FormatUtils::FORMAT_VERSION formatVersion,
         const std::vector<uint8_t *> &contentBuffers, const std::vector<int> &contentBufferSizes)
         : mHeaderBuffer(std::move(headerBuffer)), mDictBuffer(std::move(bodyBuffer)),
-          mHeaderPolicy(mHeaderBuffer->getBuffer(), formatVersion),
+          mHeaderPolicy(mHeaderBuffer->getReadOnlyByteArrayView().data(), formatVersion),
-          mExpandableHeaderBuffer(mHeaderBuffer ? mHeaderBuffer->getBuffer() : nullptr,
+          mExpandableHeaderBuffer(mHeaderBuffer->getReadWriteByteArrayView(),
-                  mHeaderPolicy.getSize(),
                   BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
-          mExpandableTrieBuffer(contentBuffers[Ver4DictConstants::TRIE_BUFFER_INDEX],
+          mExpandableTrieBuffer(
-                  contentBufferSizes[Ver4DictConstants::TRIE_BUFFER_INDEX],
+                  ReadWriteByteArrayView(contentBuffers[Ver4DictConstants::TRIE_BUFFER_INDEX],
+                          contentBufferSizes[Ver4DictConstants::TRIE_BUFFER_INDEX]),
                   BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
           mTerminalPositionLookupTable(
                   contentBuffers[Ver4DictConstants::TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX],

native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h

@@ -35,15 +35,14 @@ class BufferWithExtendableBuffer {
  public:
     static const size_t DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE;
 
-    BufferWithExtendableBuffer(uint8_t *const originalBuffer, const int originalBufferSize,
+    BufferWithExtendableBuffer(const ReadWriteByteArrayView originalBuffer,
             const int maxAdditionalBufferSize)
-            : mOriginalBuffer(originalBuffer, originalBufferSize),
+            : mOriginalBuffer(originalBuffer), mAdditionalBuffer(), mUsedAdditionalBufferSize(0),
-              mAdditionalBuffer(0), mUsedAdditionalBufferSize(0),
               mMaxAdditionalBufferSize(maxAdditionalBufferSize) {}
 
     // Without original buffer.
     BufferWithExtendableBuffer(const int maxAdditionalBufferSize)
-            : mOriginalBuffer(), mAdditionalBuffer(0), mUsedAdditionalBufferSize(0),
+            : mOriginalBuffer(), mAdditionalBuffer(), mUsedAdditionalBufferSize(0),
               mMaxAdditionalBufferSize(maxAdditionalBufferSize) {}
 
     AK_FORCE_INLINE int getTailPosition() const {

native/jni/src/suggest/policyimpl/dictionary/utils/mmapped_buffer.h

@@ -21,6 +21,7 @@
 #include <memory>
 
 #include "defines.h"
+#include "utils/byte_array_view.h"
 
 namespace latinime {
 
@@ -39,12 +40,12 @@ class MmappedBuffer {
 
     ~MmappedBuffer();
 
-    AK_FORCE_INLINE uint8_t *getBuffer() const {
+    ReadWriteByteArrayView getReadWriteByteArrayView() const {
-        return mBuffer;
+        return mByteArrayView;
     }
 
-    AK_FORCE_INLINE int getBufferSize() const {
+    ReadOnlyByteArrayView getReadOnlyByteArrayView() const {
-        return mBufferSize;
+        return mByteArrayView.getReadOnlyView();
     }
 
     AK_FORCE_INLINE bool isUpdatable() const {
@@ -55,18 +56,17 @@ class MmappedBuffer {
     AK_FORCE_INLINE MmappedBuffer(uint8_t *const buffer, const int bufferSize,
             void *const mmappedBuffer, const int alignedSize, const int mmapFd,
             const bool isUpdatable)
-            : mBuffer(buffer), mBufferSize(bufferSize), mMmappedBuffer(mmappedBuffer),
+            : mByteArrayView(buffer, bufferSize), mMmappedBuffer(mmappedBuffer),
               mAlignedSize(alignedSize), mMmapFd(mmapFd), mIsUpdatable(isUpdatable) {}
 
     // Empty file. We have to handle an empty file as a valid part of a dictionary.
     AK_FORCE_INLINE MmappedBuffer(const bool isUpdatable)
-            : mBuffer(nullptr), mBufferSize(0), mMmappedBuffer(nullptr), mAlignedSize(0),
+            : mByteArrayView(), mMmappedBuffer(nullptr), mAlignedSize(0),
               mMmapFd(0), mIsUpdatable(isUpdatable) {}
 
     DISALLOW_IMPLICIT_CONSTRUCTORS(MmappedBuffer);
 
-    uint8_t *const mBuffer;
+    const ReadWriteByteArrayView mByteArrayView;
-    const int mBufferSize;
     void *const mMmappedBuffer;
     const int mAlignedSize;
     const int mMmapFd;

native/jni/src/suggest/policyimpl/dictionary/utils/trie_map.cpp

@@ -43,9 +43,8 @@ TrieMap::TrieMap() : mBuffer(MAX_BUFFER_SIZE) {
     writeEntry(EMPTY_BITMAP_ENTRY, ROOT_BITMAP_ENTRY_INDEX);
 }
 
-TrieMap::TrieMap(uint8_t *const buffer, const int bufferSize)
+TrieMap::TrieMap(const ReadWriteByteArrayView buffer)
-        : mBuffer(buffer, bufferSize,
+        : mBuffer(buffer, BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE) {}
-                BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE) {}
 
 void TrieMap::dump(const int from, const int to) const {
     AKLOGI("BufSize: %d", mBuffer.getTailPosition());

native/jni/src/suggest/policyimpl/dictionary/utils/trie_map.h

@@ -24,6 +24,7 @@
 
 #include "defines.h"
 #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
+#include "utils/byte_array_view.h"
 
 namespace latinime {
 
@@ -161,7 +162,7 @@ class TrieMap {
 
     TrieMap();
     // Construct TrieMap using existing data in the memory region written by save().
-    TrieMap(uint8_t *const buffer, const int bufferSize);
+    TrieMap(const ReadWriteByteArrayView buffer);
     void dump(const int from = 0, const int to = 0) const;
 
     bool isNearSizeLimit() const {

native/jni/src/utils/byte_array_view.h

@@ -71,6 +71,11 @@ class ReadWriteByteArrayView {
         return ReadOnlyByteArrayView(mPtr, mSize);
     }
 
+    ReadWriteByteArrayView subView(const size_t start, const size_t n) const {
+        ASSERT(start + n <= mSize);
+        return ReadWriteByteArrayView(mPtr + start, n);
+    }
+
  private:
     DISALLOW_ASSIGNMENT_OPERATOR(ReadWriteByteArrayView);