nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge "Quit using BinaryDictionaryInfo in header reading methods."

main
Keisuke Kuroyanagi 2013-08-13 05:49:35 +00:00 committed by Android (Google) Code Review
parent 306d3a718e 99007de979
commit b40d14226b
7 changed files with 40 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.cpp
View File

@ -16,6 +16,8 @@
#include "suggest/core/dictionary/binary_dictionary_format_utils.h"
#include "suggest/core/dictionary/byte_array_utils.h"
namespace latinime {
/**
@ -27,7 +29,6 @@ const int BinaryDictionaryFormatUtils::DICTIONARY_MINIMUM_SIZE = 4;
/**
* Format versions
*/
// The versions of Latin IME that only handle format version 1 only test for the magic
// number, so we had to change it so that version 2 files would be rejected by older
// implementations. On this occasion, we made the magic number 32 bits long.

1
native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.h
View File

@ -20,7 +20,6 @@
#include <stdint.h>
#include "defines.h"
#include "suggest/core/dictionary/byte_array_utils.h"
namespace latinime {

12
native/jni/src/suggest/core/dictionary/binary_dictionary_header.cpp
View File

@ -17,7 +17,6 @@
#include "suggest/core/dictionary/binary_dictionary_header.h"
#include "defines.h"
#include "suggest/core/dictionary/binary_dictionary_info.h"
namespace latinime {
@ -26,16 +25,15 @@ const char *const BinaryDictionaryHeader::MULTIPLE_WORDS_DEMOTION_RATE_KEY =
const float BinaryDictionaryHeader::DEFAULT_MULTI_WORD_COST_MULTIPLIER = 1.0f;
const float BinaryDictionaryHeader::MULTI_WORD_COST_MULTIPLIER_SCALE = 100.0f;
BinaryDictionaryHeader::BinaryDictionaryHeader(
const BinaryDictionaryInfo *const binaryDictionaryInfo)
: mBinaryDictionaryInfo(binaryDictionaryInfo),
mDictionaryFlags(BinaryDictionaryHeaderReadingUtils::getFlags(binaryDictionaryInfo)),
mSize(BinaryDictionaryHeaderReadingUtils::getHeaderSize(binaryDictionaryInfo)),
BinaryDictionaryHeader::BinaryDictionaryHeader(const uint8_t *const dictBuf)
: mDictBuf(dictBuf),
mDictionaryFlags(BinaryDictionaryHeaderReadingUtils::getFlags(mDictBuf)),
mSize(BinaryDictionaryHeaderReadingUtils::getHeaderSize(mDictBuf)),
mMultiWordCostMultiplier(readMultiWordCostMultiplier()) {}
float BinaryDictionaryHeader::readMultiWordCostMultiplier() const {
const int headerValue = BinaryDictionaryHeaderReadingUtils::readHeaderValueInt(
mBinaryDictionaryInfo, MULTIPLE_WORDS_DEMOTION_RATE_KEY);
mDictBuf, MULTIPLE_WORDS_DEMOTION_RATE_KEY);
if (headerValue == S_INT_MIN) {
// not found
return DEFAULT_MULTI_WORD_COST_MULTIPLIER;

9
native/jni/src/suggest/core/dictionary/binary_dictionary_header.h
View File

@ -17,6 +17,8 @@
#ifndef LATINIME_BINARY_DICTIONARY_HEADER_H
#define LATINIME_BINARY_DICTIONARY_HEADER_H
#include <stdint.h>
#include "defines.h"
#include "suggest/core/dictionary/binary_dictionary_header_reading_utils.h"
@ -28,9 +30,10 @@ class BinaryDictionaryInfo;
* This class abstracts dictionary header structures and provide interface to access dictionary
* header information.
*/
// TODO:: Move header classes to policyimpl.
class BinaryDictionaryHeader {
public:
explicit BinaryDictionaryHeader(const BinaryDictionaryInfo *const binaryDictionaryInfo);
explicit BinaryDictionaryHeader(const uint8_t *const dictBuf);
AK_FORCE_INLINE int getSize() const {
return mSize;
@ -60,7 +63,7 @@ class BinaryDictionaryHeader {
outValue[0] = '\0';
return;
}
if (!BinaryDictionaryHeaderReadingUtils::readHeaderValue(mBinaryDictionaryInfo,
if (!BinaryDictionaryHeaderReadingUtils::readHeaderValue(mDictBuf,
key, outValue, outValueSize)) {
outValue[0] = '?';
outValue[1] = '\0';
@ -74,7 +77,7 @@ class BinaryDictionaryHeader {
static const float DEFAULT_MULTI_WORD_COST_MULTIPLIER;
static const float MULTI_WORD_COST_MULTIPLIER_SCALE;
const BinaryDictionaryInfo *const mBinaryDictionaryInfo;
const uint8_t *const mDictBuf;
const BinaryDictionaryHeaderReadingUtils::DictionaryFlags mDictionaryFlags;
const int mSize;
const float mMultiWordCostMultiplier;

51
native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.cpp
View File

@ -20,12 +20,11 @@
#include <cstdlib>
#include "defines.h"
#include "suggest/core/dictionary/binary_dictionary_info.h"
#include "suggest/core/dictionary/byte_array_utils.h"
namespace latinime {
const int BinaryDictionaryHeaderReadingUtils::MAX_OPTION_KEY_LENGTH = 256;
const int BinaryDictionaryHeaderReadingUtils::VERSION_2_HEADER_MAGIC_NUMBER_SIZE = 4;
const int BinaryDictionaryHeaderReadingUtils::VERSION_2_HEADER_DICTIONARY_VERSION_SIZE = 2;
const int BinaryDictionaryHeaderReadingUtils::VERSION_2_HEADER_FLAG_SIZE = 2;
@ -43,68 +42,54 @@ const BinaryDictionaryHeaderReadingUtils::DictionaryFlags
const BinaryDictionaryHeaderReadingUtils::DictionaryFlags
BinaryDictionaryHeaderReadingUtils::FRENCH_LIGATURE_PROCESSING_FLAG = 0x4;
/* static */ int BinaryDictionaryHeaderReadingUtils::getHeaderSize(
const BinaryDictionaryInfo *const binaryDictionaryInfo) {
switch (getHeaderVersion(binaryDictionaryInfo->getFormat())) {
case HEADER_VERSION_2:
// See the format of the header in the comment in
// BinaryDictionaryFormatUtils::detectFormatVersion()
return ByteArrayUtils::readUint32(binaryDictionaryInfo->getDictBuf(),
VERSION_2_HEADER_MAGIC_NUMBER_SIZE + VERSION_2_HEADER_DICTIONARY_VERSION_SIZE
+ VERSION_2_HEADER_FLAG_SIZE);
default:
return S_INT_MAX;
}
/* static */ int BinaryDictionaryHeaderReadingUtils::getHeaderSize(const uint8_t *const dictBuf) {
// See the format of the header in the comment in
// BinaryDictionaryFormatUtils::detectFormatVersion()
return ByteArrayUtils::readUint32(dictBuf,
VERSION_2_HEADER_MAGIC_NUMBER_SIZE + VERSION_2_HEADER_DICTIONARY_VERSION_SIZE
+ VERSION_2_HEADER_FLAG_SIZE);
}
/* static */ BinaryDictionaryHeaderReadingUtils::DictionaryFlags
BinaryDictionaryHeaderReadingUtils::getFlags(
const BinaryDictionaryInfo *const binaryDictionaryInfo) {
switch (getHeaderVersion(binaryDictionaryInfo->getFormat())) {
case HEADER_VERSION_2:
return ByteArrayUtils::readUint16(binaryDictionaryInfo->getDictBuf(),
VERSION_2_HEADER_MAGIC_NUMBER_SIZE + VERSION_2_HEADER_DICTIONARY_VERSION_SIZE);
default:
return NO_FLAGS;
}
BinaryDictionaryHeaderReadingUtils::getFlags(const uint8_t *const dictBuf) {
return ByteArrayUtils::readUint16(dictBuf, VERSION_2_HEADER_MAGIC_NUMBER_SIZE
+ VERSION_2_HEADER_DICTIONARY_VERSION_SIZE);
}
// Returns if the key is found or not and reads the found value into outValue.
/* static */ bool BinaryDictionaryHeaderReadingUtils::readHeaderValue(
const BinaryDictionaryInfo *const binaryDictionaryInfo,
/* static */ bool BinaryDictionaryHeaderReadingUtils::readHeaderValue(const uint8_t *const dictBuf,
const char *const key, int *outValue, const int outValueSize) {
if (outValueSize <= 0) {
return false;
}
const int headerSize = getHeaderSize(binaryDictionaryInfo);
int pos = getHeaderOptionsPosition(binaryDictionaryInfo->getFormat());
const int headerSize = getHeaderSize(dictBuf);
int pos = getHeaderOptionsPosition();
if (pos == NOT_A_DICT_POS) {
// The header doesn't have header options.
return false;
}
while (pos < headerSize) {
if(ByteArrayUtils::compareStringInBufferWithCharArray(
binaryDictionaryInfo->getDictBuf(), key, headerSize - pos, &pos) == 0) {
dictBuf, key, headerSize - pos, &pos) == 0) {
// The key was found.
const int length = ByteArrayUtils::readStringAndAdvancePosition(
binaryDictionaryInfo->getDictBuf(), outValueSize, outValue, &pos);
dictBuf, outValueSize, outValue, &pos);
// Add a 0 terminator to the string.
outValue[length < outValueSize ? length : outValueSize - 1] = '\0';
return true;
}
ByteArrayUtils::advancePositionToBehindString(
binaryDictionaryInfo->getDictBuf(), headerSize - pos, &pos);
ByteArrayUtils::advancePositionToBehindString(dictBuf, headerSize - pos, &pos);
}
// The key was not found.
return false;
}
/* static */ int BinaryDictionaryHeaderReadingUtils::readHeaderValueInt(
const BinaryDictionaryInfo *const binaryDictionaryInfo, const char *const key) {
const uint8_t *const dictBuf, const char *const key) {
const int bufferSize = LARGEST_INT_DIGIT_COUNT;
int intBuffer[bufferSize];
char charBuffer[bufferSize];
if (!readHeaderValue(binaryDictionaryInfo, key, intBuffer, bufferSize)) {
if (!readHeaderValue(dictBuf, key, intBuffer, bufferSize)) {
return S_INT_MIN;
}
for (int i = 0; i < bufferSize; ++i) {

44
native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.h
View File

@ -20,21 +20,19 @@
#include <stdint.h>
#include "defines.h"
#include "suggest/core/dictionary/binary_dictionary_format_utils.h"
namespace latinime {
class BinaryDictionaryInfo;
// TODO:: Move header classes to policyimpl.
class BinaryDictionaryHeaderReadingUtils {
public:
typedef uint16_t DictionaryFlags;
static const int MAX_OPTION_KEY_LENGTH;
static int getHeaderSize(const BinaryDictionaryInfo *const binaryDictionaryInfo);
static int getHeaderSize(const uint8_t *const dictBuf);
static DictionaryFlags getFlags(const BinaryDictionaryInfo *const binaryDictionaryInfo);
static DictionaryFlags getFlags(const uint8_t *const dictBuf);
static AK_FORCE_INLINE bool supportsDynamicUpdate(const DictionaryFlags flags) {
return (flags & SUPPORTS_DYNAMIC_UPDATE_FLAG) != 0;
@ -48,33 +46,19 @@ class BinaryDictionaryHeaderReadingUtils {
return (flags & FRENCH_LIGATURE_PROCESSING_FLAG) != 0;
}
static AK_FORCE_INLINE int getHeaderOptionsPosition(
const BinaryDictionaryFormatUtils::FORMAT_VERSION dictionaryFormat) {
switch (getHeaderVersion(dictionaryFormat)) {
case HEADER_VERSION_2:
return VERSION_2_HEADER_MAGIC_NUMBER_SIZE + VERSION_2_HEADER_DICTIONARY_VERSION_SIZE
+ VERSION_2_HEADER_FLAG_SIZE + VERSION_2_HEADER_SIZE_FIELD_SIZE;
break;
default:
return NOT_A_DICT_POS;
}
static AK_FORCE_INLINE int getHeaderOptionsPosition() {
return VERSION_2_HEADER_MAGIC_NUMBER_SIZE + VERSION_2_HEADER_DICTIONARY_VERSION_SIZE
+ VERSION_2_HEADER_FLAG_SIZE + VERSION_2_HEADER_SIZE_FIELD_SIZE;
}
static bool readHeaderValue(
const BinaryDictionaryInfo *const binaryDictionaryInfo,
static bool readHeaderValue(const uint8_t *const dictBuf,
const char *const key, int *outValue, const int outValueSize);
static int readHeaderValueInt(
const BinaryDictionaryInfo *const binaryDictionaryInfo, const char *const key);
static int readHeaderValueInt(const uint8_t *const dictBuf, const char *const key);
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(BinaryDictionaryHeaderReadingUtils);
enum HEADER_VERSION {
HEADER_VERSION_2,
UNKNOWN_HEADER_VERSION
};
static const int VERSION_2_HEADER_MAGIC_NUMBER_SIZE;
static const int VERSION_2_HEADER_DICTIONARY_VERSION_SIZE;
static const int VERSION_2_HEADER_FLAG_SIZE;
@ -88,18 +72,6 @@ class BinaryDictionaryHeaderReadingUtils {
static const DictionaryFlags SUPPORTS_DYNAMIC_UPDATE_FLAG;
static const DictionaryFlags FRENCH_LIGATURE_PROCESSING_FLAG;
static const DictionaryFlags CONTAINS_BIGRAMS_FLAG;
static HEADER_VERSION getHeaderVersion(
const BinaryDictionaryFormatUtils::FORMAT_VERSION formatVersion) {
switch(formatVersion) {
case BinaryDictionaryFormatUtils::VERSION_2:
// Fall through
case BinaryDictionaryFormatUtils::VERSION_3:
return HEADER_VERSION_2;
default:
return UNKNOWN_HEADER_VERSION;
}
}
};
}
#endif /* LATINIME_DICTIONARY_HEADER_READING_UTILS_H */

2
native/jni/src/suggest/core/dictionary/binary_dictionary_info.h
View File

@ -36,7 +36,7 @@ class BinaryDictionaryInfo {
mDictBufOffset(dictBufOffset), mIsUpdatable(isUpdatable),
mDictionaryFormat(BinaryDictionaryFormatUtils::detectFormatVersion(
mDictBuf, mDictSize)),
mDictionaryHeader(this), mDictRoot(mDictBuf + mDictionaryHeader.getSize()),
mDictionaryHeader(dictBuf), mDictRoot(mDictBuf + mDictionaryHeader.getSize()),
// TODO: Remove.
mStructurePolicy(DictionaryStructureWithBufferPolicyFactory
::newDictionaryStructurePolicy(this)) {