Merge "Read options from attributes"

main
Jean Chalard 2013-12-11 07:39:00 +00:00 committed by Android (Google) Code Review
commit 502f3aca01
8 changed files with 21 additions and 79 deletions

Binary file not shown.

View File

@ -28,11 +28,8 @@ const DigraphUtils::digraph_t DigraphUtils::GERMAN_UMLAUT_DIGRAPHS[] =
{ { 'a', 'e', 0x00E4 }, // U+00E4 : LATIN SMALL LETTER A WITH DIAERESIS { { 'a', 'e', 0x00E4 }, // U+00E4 : LATIN SMALL LETTER A WITH DIAERESIS
{ 'o', 'e', 0x00F6 }, // U+00F6 : LATIN SMALL LETTER O WITH DIAERESIS { 'o', 'e', 0x00F6 }, // U+00F6 : LATIN SMALL LETTER O WITH DIAERESIS
{ 'u', 'e', 0x00FC } }; // U+00FC : LATIN SMALL LETTER U WITH DIAERESIS { 'u', 'e', 0x00FC } }; // U+00FC : LATIN SMALL LETTER U WITH DIAERESIS
const DigraphUtils::digraph_t DigraphUtils::FRENCH_LIGATURES_DIGRAPHS[] =
{ { 'a', 'e', 0x00E6 }, // U+00E6 : LATIN SMALL LETTER AE
{ 'o', 'e', 0x0153 } }; // U+0153 : LATIN SMALL LIGATURE OE
const DigraphUtils::DigraphType DigraphUtils::USED_DIGRAPH_TYPES[] = const DigraphUtils::DigraphType DigraphUtils::USED_DIGRAPH_TYPES[] =
{ DIGRAPH_TYPE_GERMAN_UMLAUT, DIGRAPH_TYPE_FRENCH_LIGATURES }; { DIGRAPH_TYPE_GERMAN_UMLAUT };
/* static */ bool DigraphUtils::hasDigraphForCodePoint( /* static */ bool DigraphUtils::hasDigraphForCodePoint(
const DictionaryHeaderStructurePolicy *const headerPolicy, const DictionaryHeaderStructurePolicy *const headerPolicy,
@ -50,9 +47,6 @@ const DigraphUtils::DigraphType DigraphUtils::USED_DIGRAPH_TYPES[] =
if (headerPolicy->requiresGermanUmlautProcessing()) { if (headerPolicy->requiresGermanUmlautProcessing()) {
return DIGRAPH_TYPE_GERMAN_UMLAUT; return DIGRAPH_TYPE_GERMAN_UMLAUT;
} }
if (headerPolicy->requiresFrenchLigatureProcessing()) {
return DIGRAPH_TYPE_FRENCH_LIGATURES;
}
return DIGRAPH_TYPE_NONE; return DIGRAPH_TYPE_NONE;
} }
@ -86,10 +80,6 @@ const DigraphUtils::DigraphType DigraphUtils::USED_DIGRAPH_TYPES[] =
*digraphs = GERMAN_UMLAUT_DIGRAPHS; *digraphs = GERMAN_UMLAUT_DIGRAPHS;
return NELEMS(GERMAN_UMLAUT_DIGRAPHS); return NELEMS(GERMAN_UMLAUT_DIGRAPHS);
} }
if (digraphType == DIGRAPH_TYPE_FRENCH_LIGATURES) {
*digraphs = FRENCH_LIGATURES_DIGRAPHS;
return NELEMS(FRENCH_LIGATURES_DIGRAPHS);
}
return 0; return 0;
} }

View File

@ -34,7 +34,6 @@ class DigraphUtils {
typedef enum { typedef enum {
DIGRAPH_TYPE_NONE, DIGRAPH_TYPE_NONE,
DIGRAPH_TYPE_GERMAN_UMLAUT, DIGRAPH_TYPE_GERMAN_UMLAUT,
DIGRAPH_TYPE_FRENCH_LIGATURES
} DigraphType; } DigraphType;
typedef struct { int first; int second; int compositeGlyph; } digraph_t; typedef struct { int first; int second; int compositeGlyph; } digraph_t;
@ -55,7 +54,6 @@ class DigraphUtils {
const DigraphType digraphType, const int compositeGlyphCodePoint); const DigraphType digraphType, const int compositeGlyphCodePoint);
static const digraph_t GERMAN_UMLAUT_DIGRAPHS[]; static const digraph_t GERMAN_UMLAUT_DIGRAPHS[];
static const digraph_t FRENCH_LIGATURES_DIGRAPHS[];
static const DigraphType USED_DIGRAPH_TYPES[]; static const DigraphType USED_DIGRAPH_TYPES[];
}; };
} // namespace latinime } // namespace latinime

View File

@ -31,12 +31,8 @@ class DictionaryHeaderStructurePolicy {
virtual int getFormatVersionNumber() const = 0; virtual int getFormatVersionNumber() const = 0;
virtual bool supportsDynamicUpdate() const = 0;
virtual bool requiresGermanUmlautProcessing() const = 0; virtual bool requiresGermanUmlautProcessing() const = 0;
virtual bool requiresFrenchLigatureProcessing() const = 0;
virtual float getMultiWordCostMultiplier() const = 0; virtual float getMultiWordCostMultiplier() const = 0;
virtual int getLastDecayedTime() const = 0; virtual int getLastDecayedTime() const = 0;

View File

@ -20,6 +20,8 @@ namespace latinime {
// Note that these are corresponding definitions in Java side in FormatSpec.FileHeader. // Note that these are corresponding definitions in Java side in FormatSpec.FileHeader.
const char *const HeaderPolicy::MULTIPLE_WORDS_DEMOTION_RATE_KEY = "MULTIPLE_WORDS_DEMOTION_RATE"; const char *const HeaderPolicy::MULTIPLE_WORDS_DEMOTION_RATE_KEY = "MULTIPLE_WORDS_DEMOTION_RATE";
const char *const HeaderPolicy::REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY =
"REQUIRES_GERMAN_UMLAUT_PROCESSING";
// TODO: Change attribute string to "IS_DECAYING_DICT". // TODO: Change attribute string to "IS_DECAYING_DICT".
const char *const HeaderPolicy::IS_DECAYING_DICT_KEY = "USES_FORGETTING_CURVE"; const char *const HeaderPolicy::IS_DECAYING_DICT_KEY = "USES_FORGETTING_CURVE";
const char *const HeaderPolicy::LAST_UPDATED_TIME_KEY = "date"; const char *const HeaderPolicy::LAST_UPDATED_TIME_KEY = "date";
@ -66,6 +68,11 @@ float HeaderPolicy::readMultipleWordCostMultiplier() const {
return MULTIPLE_WORD_COST_MULTIPLIER_SCALE / static_cast<float>(demotionRate); return MULTIPLE_WORD_COST_MULTIPLIER_SCALE / static_cast<float>(demotionRate);
} }
bool HeaderPolicy::readRequiresGermanUmlautProcessing() const {
return HeaderReadWriteUtils::readBoolAttributeValue(&mAttributeMap,
REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY, false);
}
bool HeaderPolicy::writeHeaderToBuffer(BufferWithExtendableBuffer *const bufferToWrite, bool HeaderPolicy::writeHeaderToBuffer(BufferWithExtendableBuffer *const bufferToWrite,
const bool updatesLastUpdatedTime, const bool updatesLastDecayedTime, const bool updatesLastUpdatedTime, const bool updatesLastDecayedTime,
const int unigramCount, const int bigramCount, const int extendedRegionSize) const { const int unigramCount, const int bigramCount, const int extendedRegionSize) const {

View File

@ -36,6 +36,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
mSize(HeaderReadWriteUtils::getHeaderSize(dictBuf)), mSize(HeaderReadWriteUtils::getHeaderSize(dictBuf)),
mAttributeMap(createAttributeMapAndReadAllAttributes(dictBuf)), mAttributeMap(createAttributeMapAndReadAllAttributes(dictBuf)),
mMultiWordCostMultiplier(readMultipleWordCostMultiplier()), mMultiWordCostMultiplier(readMultipleWordCostMultiplier()),
mRequiresGermanUmlautProcessing(readRequiresGermanUmlautProcessing()),
mIsDecayingDict(HeaderReadWriteUtils::readBoolAttributeValue(&mAttributeMap, mIsDecayingDict(HeaderReadWriteUtils::readBoolAttributeValue(&mAttributeMap,
IS_DECAYING_DICT_KEY, false /* defaultValue */)), IS_DECAYING_DICT_KEY, false /* defaultValue */)),
mLastUpdatedTime(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap, mLastUpdatedTime(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
@ -58,6 +59,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
mDictionaryFlags(HeaderReadWriteUtils::createAndGetDictionaryFlagsUsingAttributeMap( mDictionaryFlags(HeaderReadWriteUtils::createAndGetDictionaryFlagsUsingAttributeMap(
attributeMap)), mSize(0), mAttributeMap(*attributeMap), attributeMap)), mSize(0), mAttributeMap(*attributeMap),
mMultiWordCostMultiplier(readMultipleWordCostMultiplier()), mMultiWordCostMultiplier(readMultipleWordCostMultiplier()),
mRequiresGermanUmlautProcessing(readRequiresGermanUmlautProcessing()),
mIsDecayingDict(HeaderReadWriteUtils::readBoolAttributeValue(&mAttributeMap, mIsDecayingDict(HeaderReadWriteUtils::readBoolAttributeValue(&mAttributeMap,
IS_DECAYING_DICT_KEY, false /* defaultValue */)), IS_DECAYING_DICT_KEY, false /* defaultValue */)),
mLastUpdatedTime(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap, mLastUpdatedTime(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
@ -71,7 +73,8 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
// Temporary dummy header. // Temporary dummy header.
HeaderPolicy() HeaderPolicy()
: mDictFormatVersion(FormatUtils::UNKNOWN_VERSION), mDictionaryFlags(0), mSize(0), : mDictFormatVersion(FormatUtils::UNKNOWN_VERSION), mDictionaryFlags(0), mSize(0),
mAttributeMap(), mMultiWordCostMultiplier(0.0f), mIsDecayingDict(false), mAttributeMap(), mMultiWordCostMultiplier(0.0f),
mRequiresGermanUmlautProcessing(false), mIsDecayingDict(false),
mLastUpdatedTime(0), mLastDecayedTime(0), mUnigramCount(0), mBigramCount(0), mLastUpdatedTime(0), mLastDecayedTime(0), mUnigramCount(0), mBigramCount(0),
mExtendedRegionSize(0), mHasHistoricalInfoOfWords(false) {} mExtendedRegionSize(0), mHasHistoricalInfoOfWords(false) {}
@ -109,18 +112,6 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
return mSize; return mSize;
} }
AK_FORCE_INLINE bool supportsDynamicUpdate() const {
return HeaderReadWriteUtils::supportsDynamicUpdate(mDictionaryFlags);
}
AK_FORCE_INLINE bool requiresGermanUmlautProcessing() const {
return HeaderReadWriteUtils::requiresGermanUmlautProcessing(mDictionaryFlags);
}
AK_FORCE_INLINE bool requiresFrenchLigatureProcessing() const {
return HeaderReadWriteUtils::requiresFrenchLigatureProcessing(mDictionaryFlags);
}
AK_FORCE_INLINE float getMultiWordCostMultiplier() const { AK_FORCE_INLINE float getMultiWordCostMultiplier() const {
return mMultiWordCostMultiplier; return mMultiWordCostMultiplier;
} }
@ -129,6 +120,10 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
return mIsDecayingDict; return mIsDecayingDict;
} }
AK_FORCE_INLINE bool requiresGermanUmlautProcessing() const {
return mRequiresGermanUmlautProcessing;
}
AK_FORCE_INLINE int getLastUpdatedTime() const { AK_FORCE_INLINE int getLastUpdatedTime() const {
return mLastUpdatedTime; return mLastUpdatedTime;
} }
@ -164,6 +159,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
DISALLOW_COPY_AND_ASSIGN(HeaderPolicy); DISALLOW_COPY_AND_ASSIGN(HeaderPolicy);
static const char *const MULTIPLE_WORDS_DEMOTION_RATE_KEY; static const char *const MULTIPLE_WORDS_DEMOTION_RATE_KEY;
static const char *const REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY;
static const char *const IS_DECAYING_DICT_KEY; static const char *const IS_DECAYING_DICT_KEY;
static const char *const LAST_UPDATED_TIME_KEY; static const char *const LAST_UPDATED_TIME_KEY;
static const char *const LAST_DECAYED_TIME_KEY; static const char *const LAST_DECAYED_TIME_KEY;
@ -179,6 +175,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
const int mSize; const int mSize;
HeaderReadWriteUtils::AttributeMap mAttributeMap; HeaderReadWriteUtils::AttributeMap mAttributeMap;
const float mMultiWordCostMultiplier; const float mMultiWordCostMultiplier;
const bool mRequiresGermanUmlautProcessing;
const bool mIsDecayingDict; const bool mIsDecayingDict;
const int mLastUpdatedTime; const int mLastUpdatedTime;
const int mLastDecayedTime; const int mLastDecayedTime;
@ -188,6 +185,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
const bool mHasHistoricalInfoOfWords; const bool mHasHistoricalInfoOfWords;
float readMultipleWordCostMultiplier() const; float readMultipleWordCostMultiplier() const;
bool readRequiresGermanUmlautProcessing() const;
static HeaderReadWriteUtils::AttributeMap createAttributeMapAndReadAllAttributes( static HeaderReadWriteUtils::AttributeMap createAttributeMapAndReadAllAttributes(
const uint8_t *const dictBuf); const uint8_t *const dictBuf);

View File

@ -35,22 +35,6 @@ const int HeaderReadWriteUtils::HEADER_FLAG_SIZE = 2;
const int HeaderReadWriteUtils::HEADER_SIZE_FIELD_SIZE = 4; const int HeaderReadWriteUtils::HEADER_SIZE_FIELD_SIZE = 4;
const HeaderReadWriteUtils::DictionaryFlags HeaderReadWriteUtils::NO_FLAGS = 0; const HeaderReadWriteUtils::DictionaryFlags HeaderReadWriteUtils::NO_FLAGS = 0;
// Flags for special processing
// Those *must* match the flags in makedict (FormatSpec#*_PROCESSING_FLAG) or
// something very bad (like, the apocalypse) will happen. Please update both at the same time.
const HeaderReadWriteUtils::DictionaryFlags
HeaderReadWriteUtils::GERMAN_UMLAUT_PROCESSING_FLAG = 0x1;
const HeaderReadWriteUtils::DictionaryFlags
HeaderReadWriteUtils::SUPPORTS_DYNAMIC_UPDATE_FLAG = 0x2;
const HeaderReadWriteUtils::DictionaryFlags
HeaderReadWriteUtils::FRENCH_LIGATURE_PROCESSING_FLAG = 0x4;
// Note that these are corresponding definitions in Java side in FormatSpec.FileHeader.
const char *const HeaderReadWriteUtils::SUPPORTS_DYNAMIC_UPDATE_KEY = "SUPPORTS_DYNAMIC_UPDATE";
const char *const HeaderReadWriteUtils::REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY =
"REQUIRES_GERMAN_UMLAUT_PROCESSING";
const char *const HeaderReadWriteUtils::REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY =
"REQUIRES_FRENCH_LIGATURE_PROCESSING";
/* static */ int HeaderReadWriteUtils::getHeaderSize(const uint8_t *const dictBuf) { /* static */ int HeaderReadWriteUtils::getHeaderSize(const uint8_t *const dictBuf) {
// See the format of the header in the comment in // See the format of the header in the comment in
@ -68,17 +52,7 @@ const char *const HeaderReadWriteUtils::REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY
/* static */ HeaderReadWriteUtils::DictionaryFlags /* static */ HeaderReadWriteUtils::DictionaryFlags
HeaderReadWriteUtils::createAndGetDictionaryFlagsUsingAttributeMap( HeaderReadWriteUtils::createAndGetDictionaryFlagsUsingAttributeMap(
const HeaderReadWriteUtils::AttributeMap *const attributeMap) { const HeaderReadWriteUtils::AttributeMap *const attributeMap) {
const bool requiresGermanUmlautProcessing = readBoolAttributeValue(attributeMap, return NO_FLAGS;
REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY, false /* defaultValue */);
const bool requiresFrenchLigatureProcessing = readBoolAttributeValue(attributeMap,
REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY, false /* defaultValue */);
const bool supportsDynamicUpdate = readBoolAttributeValue(attributeMap,
SUPPORTS_DYNAMIC_UPDATE_KEY, false /* defaultValue */);
DictionaryFlags dictflags = NO_FLAGS;
dictflags |= requiresGermanUmlautProcessing ? GERMAN_UMLAUT_PROCESSING_FLAG : 0;
dictflags |= requiresFrenchLigatureProcessing ? FRENCH_LIGATURE_PROCESSING_FLAG : 0;
dictflags |= supportsDynamicUpdate ? SUPPORTS_DYNAMIC_UPDATE_FLAG : 0;
return dictflags;
} }
/* static */ void HeaderReadWriteUtils::fetchAllHeaderAttributes(const uint8_t *const dictBuf, /* static */ void HeaderReadWriteUtils::fetchAllHeaderAttributes(const uint8_t *const dictBuf,

View File

@ -37,18 +37,6 @@ class HeaderReadWriteUtils {
static DictionaryFlags getFlags(const uint8_t *const dictBuf); static DictionaryFlags getFlags(const uint8_t *const dictBuf);
static AK_FORCE_INLINE bool supportsDynamicUpdate(const DictionaryFlags flags) {
return (flags & SUPPORTS_DYNAMIC_UPDATE_FLAG) != 0;
}
static AK_FORCE_INLINE bool requiresGermanUmlautProcessing(const DictionaryFlags flags) {
return (flags & GERMAN_UMLAUT_PROCESSING_FLAG) != 0;
}
static AK_FORCE_INLINE bool requiresFrenchLigatureProcessing(const DictionaryFlags flags) {
return (flags & FRENCH_LIGATURE_PROCESSING_FLAG) != 0;
}
static AK_FORCE_INLINE int getHeaderOptionsPosition() { static AK_FORCE_INLINE int getHeaderOptionsPosition() {
return HEADER_MAGIC_NUMBER_SIZE + HEADER_DICTIONARY_VERSION_SIZE + HEADER_FLAG_SIZE return HEADER_MAGIC_NUMBER_SIZE + HEADER_DICTIONARY_VERSION_SIZE + HEADER_FLAG_SIZE
+ HEADER_SIZE_FIELD_SIZE; + HEADER_SIZE_FIELD_SIZE;
@ -101,17 +89,8 @@ class HeaderReadWriteUtils {
static const int HEADER_FLAG_SIZE; static const int HEADER_FLAG_SIZE;
static const int HEADER_SIZE_FIELD_SIZE; static const int HEADER_SIZE_FIELD_SIZE;
// Value for the "flags" field. It's unused at the moment.
static const DictionaryFlags NO_FLAGS; static const DictionaryFlags NO_FLAGS;
// Flags for special processing
// Those *must* match the flags in makedict (FormatSpec#*_PROCESSING_FLAGS) or
// something very bad (like, the apocalypse) will happen. Please update both at the same time.
static const DictionaryFlags GERMAN_UMLAUT_PROCESSING_FLAG;
static const DictionaryFlags SUPPORTS_DYNAMIC_UPDATE_FLAG;
static const DictionaryFlags FRENCH_LIGATURE_PROCESSING_FLAG;
static const char *const SUPPORTS_DYNAMIC_UPDATE_KEY;
static const char *const REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY;
static const char *const REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY;
static void setIntAttributeInner(AttributeMap *const headerAttributes, static void setIntAttributeInner(AttributeMap *const headerAttributes,
const AttributeMap::key_type *const key, const int value); const AttributeMap::key_type *const key, const int value);