nyancrimew/LatinIME
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

am 2a41a356: Add code point writing methods for updatable dictionaries.

Browse source 
* commit '2a41a35635be0d1b51ecc84ac03b503f0b9a14a3':
  Add code point writing methods for updatable dictionaries.
This commit is contained in:
Keisuke Kuroyanagi 2013-09-04 22:29:36 -07:00 committed by Android Git Automerger
commit b646117398
3 changed files with 118 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

66
native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.cpp
View file

@ -22,4 +22,70 @@ const size_t BufferWithExtendableBuffer::INITIAL_ADDITIONAL_BUFFER_SIZE = 16 * 1
const size_t BufferWithExtendableBuffer::MAX_ADDITIONAL_BUFFER_SIZE = 1024 * 1024;
const size_t BufferWithExtendableBuffer::EXTEND_ADDITIONAL_BUFFER_SIZE_STEP = 16 * 1024;
bool BufferWithExtendableBuffer::writeUintAndAdvancePosition(const uint32_t data, const int size,
int *const pos) {
if (!(size >= 1 && size <= 4)) {
AKLOGI("writeUintAndAdvancePosition() is called with invalid size: %d", size);
ASSERT(false);
return false;
}
if (!checkAndPrepareWriting(*pos, size)) {
return false;
}
const bool usesAdditionalBuffer = isInAdditionalBuffer(*pos);
uint8_t *const buffer = usesAdditionalBuffer ? &mAdditionalBuffer[0] : mOriginalBuffer;
if (usesAdditionalBuffer) {
*pos -= mOriginalBufferSize;
}
ByteArrayUtils::writeUintAndAdvancePosition(buffer, data, size, pos);
if (usesAdditionalBuffer) {
*pos += mOriginalBufferSize;
}
return true;
}
bool BufferWithExtendableBuffer::writeCodePointsAndAdvancePosition(const int *const codePoints,
const int codePointCount, const bool writesTerminator ,int *const pos) {
const size_t size = ByteArrayUtils::calculateRequiredByteCountToStoreCodePoints(
codePoints, codePointCount, writesTerminator);
if (!checkAndPrepareWriting(*pos, size)) {
return false;
}
const bool usesAdditionalBuffer = isInAdditionalBuffer(*pos);
uint8_t *const buffer = usesAdditionalBuffer ? &mAdditionalBuffer[0] : mOriginalBuffer;
if (usesAdditionalBuffer) {
*pos -= mOriginalBufferSize;
}
ByteArrayUtils::writeCodePointsAndAdvancePosition(buffer, codePoints, codePointCount,
writesTerminator, pos);
if (usesAdditionalBuffer) {
*pos += mOriginalBufferSize;
}
return true;
}
bool BufferWithExtendableBuffer::checkAndPrepareWriting(const int pos, const int size) {
if (isInAdditionalBuffer(pos)) {
if (pos == mUsedAdditionalBufferSize) {
// Append data to the tail.
if (pos + size > static_cast<int>(mAdditionalBuffer.size())) {
// Need to extend buffer.
if (!extendBuffer()) {
return false;
}
}
mUsedAdditionalBufferSize += size;
} else if (pos + size >= mUsedAdditionalBufferSize) {
// The access will beyond the tail of used region.
return false;
}
} else {
if (pos < 0 || mOriginalBufferSize < pos + size) {
// Invalid position or violate the boundary.
return false;
}
}
return true;
}
}

49
native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h
View file

@ -66,27 +66,10 @@ class BufferWithExtendableBuffer {
* Writing is allowed for original buffer, already written region of additional buffer and the
* tail of additional buffer.
*/
AK_FORCE_INLINE bool writeUintAndAdvancePosition(const uint32_t data, const int size,
int *const pos) {
if (!(size >= 1 && size <= 4)) {
AKLOGI("writeUintAndAdvancePosition() is called with invalid size: %d", size);
ASSERT(false);
return false;
}
if (!checkAndPrepareWriting(*pos, size)) {
return false;
}
const bool usesAdditionalBuffer = isInAdditionalBuffer(*pos);
uint8_t *const buffer = usesAdditionalBuffer ? &mAdditionalBuffer[0] : mOriginalBuffer;
if (usesAdditionalBuffer) {
*pos -= mOriginalBufferSize;
}
ByteArrayUtils::writeUintAndAdvancePosition(buffer, data, size, pos);
if (usesAdditionalBuffer) {
*pos += mOriginalBufferSize;
}
return true;
}
bool writeUintAndAdvancePosition(const uint32_t data, const int size, int *const pos);
bool writeCodePointsAndAdvancePosition(const int *const codePoints, const int codePointCount,
const bool writesTerminator, int *const pos);
private:
DISALLOW_COPY_AND_ASSIGN(BufferWithExtendableBuffer);
@ -112,29 +95,7 @@ class BufferWithExtendableBuffer {
// Returns if it is possible to write size-bytes from pos. When pos is at the tail position of
// the additional buffer, try extending the buffer.
AK_FORCE_INLINE bool checkAndPrepareWriting(const int pos, const int size) {
if (isInAdditionalBuffer(pos)) {
if (pos == mUsedAdditionalBufferSize) {
// Append data to the tail.
if (pos + size > static_cast<int>(mAdditionalBuffer.size())) {
// Need to extend buffer.
if (!extendBuffer()) {
return false;
}
}
mUsedAdditionalBufferSize += size;
} else if (pos + size >= mUsedAdditionalBufferSize) {
// The access will beyond the tail of used region.
return false;
}
} else {
if (pos < 0 || mOriginalBufferSize < pos + size) {
// Invalid position or violate the boundary.
return false;
}
}
return true;
}
AK_FORCE_INLINE bool checkAndPrepareWriting(const int pos, const int size);
};
}
#endif /* LATINIME_BUFFER_WITH_EXTENDABLE_BUFFER_H */

49
native/jni/src/suggest/policyimpl/dictionary/utils/byte_array_utils.h
View file

@ -115,7 +115,7 @@ class ByteArrayUtils {
}
/**
* Code Point
* Code Point Reading
*
* 1 byte = bbbbbbbb match
* case 000xxxxx: xxxxx << 16 + next byte << 8 + next byte
@ -149,7 +149,7 @@ class ByteArrayUtils {
}
/**
* String (array of code points)
* String (array of code points) Reading
*
* Reads code points until the terminator is found.
*/
@ -176,6 +176,51 @@ class ByteArrayUtils {
return length;
}
/**
* String (array of code points) Writing
*/
static void writeCodePointsAndAdvancePosition(uint8_t *const buffer,
const int *const codePoints, const int codePointCount, const bool writesTerminator,
int *const pos) {
for (int i = 0; i < codePointCount; ++i) {
const int codePoint = codePoints[i];
if (codePoint == NOT_A_CODE_POINT || codePoint == CHARACTER_ARRAY_TERMINATOR) {
break;
} else if (codePoint < MINIMAL_ONE_BYTE_CHARACTER_VALUE) {
// three bytes character.
writeUint24AndAdvancePosition(buffer, codePoint, pos);
} else {
// one byte character.
writeUint8AndAdvancePosition(buffer, codePoint, pos);
}
}
if (writesTerminator) {
writeUint8AndAdvancePosition(buffer, CHARACTER_ARRAY_TERMINATOR, pos);
}
}
static int calculateRequiredByteCountToStoreCodePoints(const int *const codePoints,
const int codePointCount, const bool writesTerminator) {
int byteCount = 0;
for (int i = 0; i < codePointCount; ++i) {
const int codePoint = codePoints[i];
if (codePoint == NOT_A_CODE_POINT || codePoint == CHARACTER_ARRAY_TERMINATOR) {
break;
} else if (codePoint < MINIMAL_ONE_BYTE_CHARACTER_VALUE) {
// three bytes character.
byteCount += 3;
} else {
// one byte character.
byteCount += 1;
}
}
if (writesTerminator) {
// The terminator is one byte.
byteCount += 1;
}
return byteCount;
}
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(ByteArrayUtils);