nyancrimew/LatinIME
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge "Read options from attributes"

Browse source
This commit is contained in:
Jean Chalard 2013-12-11 07:39:00 +00:00 committed by Android (Google) Code Review
commit 502f3aca01
8 changed files with 21 additions and 79 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

BIN
java/res/raw/main_de.dict
View file

Binary file not shown.

12
native/jni/src/suggest/core/dictionary/digraph_utils.cpp
View file

@ -28,11 +28,8 @@ const DigraphUtils::digraph_t DigraphUtils::GERMAN_UMLAUT_DIGRAPHS[] =
{ { 'a', 'e', 0x00E4 }, // U+00E4 : LATIN SMALL LETTER A WITH DIAERESIS
{ 'o', 'e', 0x00F6 }, // U+00F6 : LATIN SMALL LETTER O WITH DIAERESIS
{ 'u', 'e', 0x00FC } }; // U+00FC : LATIN SMALL LETTER U WITH DIAERESIS
const DigraphUtils::digraph_t DigraphUtils::FRENCH_LIGATURES_DIGRAPHS[] =
{ { 'a', 'e', 0x00E6 }, // U+00E6 : LATIN SMALL LETTER AE
{ 'o', 'e', 0x0153 } }; // U+0153 : LATIN SMALL LIGATURE OE
const DigraphUtils::DigraphType DigraphUtils::USED_DIGRAPH_TYPES[] =
{ DIGRAPH_TYPE_GERMAN_UMLAUT, DIGRAPH_TYPE_FRENCH_LIGATURES };
{ DIGRAPH_TYPE_GERMAN_UMLAUT };
/* static */ bool DigraphUtils::hasDigraphForCodePoint(
const DictionaryHeaderStructurePolicy *const headerPolicy,
@ -50,9 +47,6 @@ const DigraphUtils::DigraphType DigraphUtils::USED_DIGRAPH_TYPES[] =
if (headerPolicy->requiresGermanUmlautProcessing()) {
return DIGRAPH_TYPE_GERMAN_UMLAUT;
}
if (headerPolicy->requiresFrenchLigatureProcessing()) {
return DIGRAPH_TYPE_FRENCH_LIGATURES;
}
return DIGRAPH_TYPE_NONE;
}
@ -86,10 +80,6 @@ const DigraphUtils::DigraphType DigraphUtils::USED_DIGRAPH_TYPES[] =
*digraphs = GERMAN_UMLAUT_DIGRAPHS;
return NELEMS(GERMAN_UMLAUT_DIGRAPHS);
}
if (digraphType == DIGRAPH_TYPE_FRENCH_LIGATURES) {
*digraphs = FRENCH_LIGATURES_DIGRAPHS;
return NELEMS(FRENCH_LIGATURES_DIGRAPHS);
}
return 0;
}

2
native/jni/src/suggest/core/dictionary/digraph_utils.h
View file

@ -34,7 +34,6 @@ class DigraphUtils {
typedef enum {
DIGRAPH_TYPE_NONE,
DIGRAPH_TYPE_GERMAN_UMLAUT,
DIGRAPH_TYPE_FRENCH_LIGATURES
} DigraphType;
typedef struct { int first; int second; int compositeGlyph; } digraph_t;
@ -55,7 +54,6 @@ class DigraphUtils {
const DigraphType digraphType, const int compositeGlyphCodePoint);
static const digraph_t GERMAN_UMLAUT_DIGRAPHS[];
static const digraph_t FRENCH_LIGATURES_DIGRAPHS[];
static const DigraphType USED_DIGRAPH_TYPES[];
};
} // namespace latinime

4
native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h
View file

@ -31,12 +31,8 @@ class DictionaryHeaderStructurePolicy {
virtual int getFormatVersionNumber() const = 0;
virtual bool supportsDynamicUpdate() const = 0;
virtual bool requiresGermanUmlautProcessing() const = 0;
virtual bool requiresFrenchLigatureProcessing() const = 0;
virtual float getMultiWordCostMultiplier() const = 0;
virtual int getLastDecayedTime() const = 0;

7
native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp
View file

@ -20,6 +20,8 @@ namespace latinime {
// Note that these are corresponding definitions in Java side in FormatSpec.FileHeader.
const char *const HeaderPolicy::MULTIPLE_WORDS_DEMOTION_RATE_KEY = "MULTIPLE_WORDS_DEMOTION_RATE";
const char *const HeaderPolicy::REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY =
"REQUIRES_GERMAN_UMLAUT_PROCESSING";
// TODO: Change attribute string to "IS_DECAYING_DICT".
const char *const HeaderPolicy::IS_DECAYING_DICT_KEY = "USES_FORGETTING_CURVE";
const char *const HeaderPolicy::LAST_UPDATED_TIME_KEY = "date";
@ -66,6 +68,11 @@ float HeaderPolicy::readMultipleWordCostMultiplier() const {
return MULTIPLE_WORD_COST_MULTIPLIER_SCALE / static_cast<float>(demotionRate);
}
bool HeaderPolicy::readRequiresGermanUmlautProcessing() const {
return HeaderReadWriteUtils::readBoolAttributeValue(&mAttributeMap,
REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY, false);
}
bool HeaderPolicy::writeHeaderToBuffer(BufferWithExtendableBuffer *const bufferToWrite,
const bool updatesLastUpdatedTime, const bool updatesLastDecayedTime,
const int unigramCount, const int bigramCount, const int extendedRegionSize) const {

24
native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
View file

@ -36,6 +36,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
mSize(HeaderReadWriteUtils::getHeaderSize(dictBuf)),
mAttributeMap(createAttributeMapAndReadAllAttributes(dictBuf)),
mMultiWordCostMultiplier(readMultipleWordCostMultiplier()),
mRequiresGermanUmlautProcessing(readRequiresGermanUmlautProcessing()),
mIsDecayingDict(HeaderReadWriteUtils::readBoolAttributeValue(&mAttributeMap,
IS_DECAYING_DICT_KEY, false /* defaultValue */)),
mLastUpdatedTime(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
@ -58,6 +59,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
mDictionaryFlags(HeaderReadWriteUtils::createAndGetDictionaryFlagsUsingAttributeMap(
attributeMap)), mSize(0), mAttributeMap(*attributeMap),
mMultiWordCostMultiplier(readMultipleWordCostMultiplier()),
mRequiresGermanUmlautProcessing(readRequiresGermanUmlautProcessing()),
mIsDecayingDict(HeaderReadWriteUtils::readBoolAttributeValue(&mAttributeMap,
IS_DECAYING_DICT_KEY, false /* defaultValue */)),
mLastUpdatedTime(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
@ -71,7 +73,8 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
// Temporary dummy header.
HeaderPolicy()
: mDictFormatVersion(FormatUtils::UNKNOWN_VERSION), mDictionaryFlags(0), mSize(0),
mAttributeMap(), mMultiWordCostMultiplier(0.0f), mIsDecayingDict(false),
mAttributeMap(), mMultiWordCostMultiplier(0.0f),
mRequiresGermanUmlautProcessing(false), mIsDecayingDict(false),
mLastUpdatedTime(0), mLastDecayedTime(0), mUnigramCount(0), mBigramCount(0),
mExtendedRegionSize(0), mHasHistoricalInfoOfWords(false) {}
@ -109,18 +112,6 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
return mSize;
}
AK_FORCE_INLINE bool supportsDynamicUpdate() const {
return HeaderReadWriteUtils::supportsDynamicUpdate(mDictionaryFlags);
}
AK_FORCE_INLINE bool requiresGermanUmlautProcessing() const {
return HeaderReadWriteUtils::requiresGermanUmlautProcessing(mDictionaryFlags);
}
AK_FORCE_INLINE bool requiresFrenchLigatureProcessing() const {
return HeaderReadWriteUtils::requiresFrenchLigatureProcessing(mDictionaryFlags);
}
AK_FORCE_INLINE float getMultiWordCostMultiplier() const {
return mMultiWordCostMultiplier;
}
@ -129,6 +120,10 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
return mIsDecayingDict;
}
AK_FORCE_INLINE bool requiresGermanUmlautProcessing() const {
return mRequiresGermanUmlautProcessing;
}
AK_FORCE_INLINE int getLastUpdatedTime() const {
return mLastUpdatedTime;
}
@ -164,6 +159,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
DISALLOW_COPY_AND_ASSIGN(HeaderPolicy);
static const char *const MULTIPLE_WORDS_DEMOTION_RATE_KEY;
static const char *const REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY;
static const char *const IS_DECAYING_DICT_KEY;
static const char *const LAST_UPDATED_TIME_KEY;
static const char *const LAST_DECAYED_TIME_KEY;
@ -179,6 +175,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
const int mSize;
HeaderReadWriteUtils::AttributeMap mAttributeMap;
const float mMultiWordCostMultiplier;
const bool mRequiresGermanUmlautProcessing;
const bool mIsDecayingDict;
const int mLastUpdatedTime;
const int mLastDecayedTime;
@ -188,6 +185,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
const bool mHasHistoricalInfoOfWords;
float readMultipleWordCostMultiplier() const;
bool readRequiresGermanUmlautProcessing() const;
static HeaderReadWriteUtils::AttributeMap createAttributeMapAndReadAllAttributes(
const uint8_t *const dictBuf);

28
native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp
View file

@ -35,22 +35,6 @@ const int HeaderReadWriteUtils::HEADER_FLAG_SIZE = 2;
const int HeaderReadWriteUtils::HEADER_SIZE_FIELD_SIZE = 4;
const HeaderReadWriteUtils::DictionaryFlags HeaderReadWriteUtils::NO_FLAGS = 0;
// Flags for special processing
// Those *must* match the flags in makedict (FormatSpec#*_PROCESSING_FLAG) or
// something very bad (like, the apocalypse) will happen. Please update both at the same time.
const HeaderReadWriteUtils::DictionaryFlags
HeaderReadWriteUtils::GERMAN_UMLAUT_PROCESSING_FLAG = 0x1;
const HeaderReadWriteUtils::DictionaryFlags
HeaderReadWriteUtils::SUPPORTS_DYNAMIC_UPDATE_FLAG = 0x2;
const HeaderReadWriteUtils::DictionaryFlags
HeaderReadWriteUtils::FRENCH_LIGATURE_PROCESSING_FLAG = 0x4;
// Note that these are corresponding definitions in Java side in FormatSpec.FileHeader.
const char *const HeaderReadWriteUtils::SUPPORTS_DYNAMIC_UPDATE_KEY = "SUPPORTS_DYNAMIC_UPDATE";
const char *const HeaderReadWriteUtils::REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY =
"REQUIRES_GERMAN_UMLAUT_PROCESSING";
const char *const HeaderReadWriteUtils::REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY =
"REQUIRES_FRENCH_LIGATURE_PROCESSING";
/* static */ int HeaderReadWriteUtils::getHeaderSize(const uint8_t *const dictBuf) {
// See the format of the header in the comment in
@ -68,17 +52,7 @@ const char *const HeaderReadWriteUtils::REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY
/* static */ HeaderReadWriteUtils::DictionaryFlags
HeaderReadWriteUtils::createAndGetDictionaryFlagsUsingAttributeMap(
const HeaderReadWriteUtils::AttributeMap *const attributeMap) {
const bool requiresGermanUmlautProcessing = readBoolAttributeValue(attributeMap,
REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY, false /* defaultValue */);
const bool requiresFrenchLigatureProcessing = readBoolAttributeValue(attributeMap,
REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY, false /* defaultValue */);
const bool supportsDynamicUpdate = readBoolAttributeValue(attributeMap,
SUPPORTS_DYNAMIC_UPDATE_KEY, false /* defaultValue */);
DictionaryFlags dictflags = NO_FLAGS;
dictflags |= requiresGermanUmlautProcessing ? GERMAN_UMLAUT_PROCESSING_FLAG : 0;
dictflags |= requiresFrenchLigatureProcessing ? FRENCH_LIGATURE_PROCESSING_FLAG : 0;
dictflags |= supportsDynamicUpdate ? SUPPORTS_DYNAMIC_UPDATE_FLAG : 0;
return dictflags;
return NO_FLAGS;
}
/* static */ void HeaderReadWriteUtils::fetchAllHeaderAttributes(const uint8_t *const dictBuf,

23
native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.h
View file

@ -37,18 +37,6 @@ class HeaderReadWriteUtils {
static DictionaryFlags getFlags(const uint8_t *const dictBuf);
static AK_FORCE_INLINE bool supportsDynamicUpdate(const DictionaryFlags flags) {
return (flags & SUPPORTS_DYNAMIC_UPDATE_FLAG) != 0;
}
static AK_FORCE_INLINE bool requiresGermanUmlautProcessing(const DictionaryFlags flags) {
return (flags & GERMAN_UMLAUT_PROCESSING_FLAG) != 0;
}
static AK_FORCE_INLINE bool requiresFrenchLigatureProcessing(const DictionaryFlags flags) {
return (flags & FRENCH_LIGATURE_PROCESSING_FLAG) != 0;
}
static AK_FORCE_INLINE int getHeaderOptionsPosition() {
return HEADER_MAGIC_NUMBER_SIZE + HEADER_DICTIONARY_VERSION_SIZE + HEADER_FLAG_SIZE
+ HEADER_SIZE_FIELD_SIZE;
@ -101,17 +89,8 @@ class HeaderReadWriteUtils {
static const int HEADER_FLAG_SIZE;
static const int HEADER_SIZE_FIELD_SIZE;
// Value for the "flags" field. It's unused at the moment.
static const DictionaryFlags NO_FLAGS;
// Flags for special processing
// Those *must* match the flags in makedict (FormatSpec#*_PROCESSING_FLAGS) or
// something very bad (like, the apocalypse) will happen. Please update both at the same time.
static const DictionaryFlags GERMAN_UMLAUT_PROCESSING_FLAG;
static const DictionaryFlags SUPPORTS_DYNAMIC_UPDATE_FLAG;
static const DictionaryFlags FRENCH_LIGATURE_PROCESSING_FLAG;
static const char *const SUPPORTS_DYNAMIC_UPDATE_KEY;
static const char *const REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY;
static const char *const REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY;
static void setIntAttributeInner(AttributeMap *const headerAttributes,
const AttributeMap::key_type *const key, const int value);