nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge "Regenerate old version dictionaries using new format."

main
Keisuke Kuroynagi 2013-07-01 07:16:15 +00:00 committed by Android (Google) Code Review
parent 70ee2201a0 a62b5b22ef
commit 109ba3ace3
6 changed files with 13 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java
View File

@ -224,14 +224,10 @@ final public class BinaryDictionaryGetter {
}
}
// ## HACK ## we prevent usage of a dictionary before version 18 for English only. The reason
// for this is, since those do not include whitelist entries, the new code with an old version
// of the dictionary would lose whitelist functionality.
// ## HACK ## we prevent usage of a dictionary before version 18. The reason for this is, since
// those do not include whitelist entries, the new code with an old version of the dictionary
// would lose whitelist functionality.
private static boolean hackCanUseDictionaryFile(final Locale locale, final File f) {
// Only for English - other languages didn't have a whitelist, hence this
// ad-hoc ## HACK ##
if (!Locale.ENGLISH.getLanguage().equals(locale.getLanguage())) return true;
FileInputStream inStream = null;
try {
// Read the version of the file

9
java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
View File

@ -92,7 +92,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
/** Controls access to the local binary dictionary for this instance. */
private final DictionaryController mLocalDictionaryController = new DictionaryController();
private static final int BINARY_DICT_VERSION = 1;
// TODO: Regenerate version 3 binary dictionary.
private static final int BINARY_DICT_VERSION = 2;
private static final FormatSpec.FormatOptions FORMAT_OPTIONS =
new FormatSpec.FormatOptions(BINARY_DICT_VERSION);
@ -415,6 +416,12 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
// shared dictionary.
loadBinaryDictionary();
}
if (mBinaryDictionary != null && !mBinaryDictionary.isValidDictionary()) {
// Binary dictionary is not valid. Regenerate the dictionary file.
mSharedDictionaryController.mLastUpdateTime = time;
generateBinaryDictionary();
loadBinaryDictionary();
}
mLocalDictionaryController.mLastUpdateTime = time;
} finally {
mSharedDictionaryController.unlock();

10
native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.cpp
View File

@ -27,10 +27,6 @@ const int BinaryDictionaryFormatUtils::DICTIONARY_MINIMUM_SIZE = 4;
/**
* Format versions
*/
// Originally, format version 1 had a 16-bit magic number, then the version number `01'
// then options that must be 0. Hence the first 32-bits of the format are always as follow
// and it's okay to consider them a magic number as a whole.
const uint32_t BinaryDictionaryFormatUtils::FORMAT_VERSION_1_MAGIC_NUMBER = 0x78B10100;
// The versions of Latin IME that only handle format version 1 only test for the magic
// number, so we had to change it so that version 2 files would be rejected by older
@ -50,12 +46,6 @@ const int BinaryDictionaryFormatUtils::FORMAT_VERSION_2_MINIMUM_SIZE = 12;
}
const uint32_t magicNumber = ByteArrayUtils::readUint32(dict, 0);
switch (magicNumber) {
case FORMAT_VERSION_1_MAGIC_NUMBER:
// Format 1 header is exactly 5 bytes long and looks like:
// Magic number (2 bytes) 0x78 0xB1
// Version number (1 byte) 0x01
// Options (2 bytes) must be 0x00 0x00
return VERSION_1;
case FORMAT_VERSION_2_MAGIC_NUMBER:
// Version 2 dictionaries are at least 12 bytes long.
// If this dictionary has the version 2 magic number but is less than 12 bytes long,

6
native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.h
View File

@ -33,10 +33,9 @@ namespace latinime {
*/
class BinaryDictionaryFormatUtils {
public:
// TODO: Remove obsolete version logic
// TODO: Support version 3 format.
enum FORMAT_VERSION {
VERSION_1,
VERSION_2,
VERSION_2 = 1,
UNKNOWN_VERSION
};
@ -46,7 +45,6 @@ class BinaryDictionaryFormatUtils {
DISALLOW_IMPLICIT_CONSTRUCTORS(BinaryDictionaryFormatUtils);
static const int DICTIONARY_MINIMUM_SIZE;
static const uint32_t FORMAT_VERSION_1_MAGIC_NUMBER;
static const uint32_t FORMAT_VERSION_2_MAGIC_NUMBER;
static const int FORMAT_VERSION_2_MINIMUM_SIZE;
};

6
native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.cpp
View File

@ -26,8 +26,6 @@ namespace latinime {
const int BinaryDictionaryHeaderReadingUtils::MAX_OPTION_KEY_LENGTH = 256;
const int BinaryDictionaryHeaderReadingUtils::FORMAT_VERSION_1_HEADER_SIZE = 5;
const int BinaryDictionaryHeaderReadingUtils::VERSION_2_MAGIC_NUMBER_SIZE = 4;
const int BinaryDictionaryHeaderReadingUtils::VERSION_2_DICTIONARY_VERSION_SIZE = 2;
const int BinaryDictionaryHeaderReadingUtils::VERSION_2_DICTIONARY_FLAG_SIZE = 2;
@ -48,8 +46,6 @@ const BinaryDictionaryHeaderReadingUtils::DictionaryFlags
/* static */ int BinaryDictionaryHeaderReadingUtils::getHeaderSize(
const BinaryDictionaryInfo *const binaryDictionaryInfo) {
switch (binaryDictionaryInfo->getFormat()) {
case BinaryDictionaryFormatUtils::VERSION_1:
return FORMAT_VERSION_1_HEADER_SIZE;
case BinaryDictionaryFormatUtils::VERSION_2:
// See the format of the header in the comment in
// BinaryDictionaryFormatUtils::detectFormatVersion()
@ -65,8 +61,6 @@ const BinaryDictionaryHeaderReadingUtils::DictionaryFlags
BinaryDictionaryHeaderReadingUtils::getFlags(
const BinaryDictionaryInfo *const binaryDictionaryInfo) {
switch (binaryDictionaryInfo->getFormat()) {
case BinaryDictionaryFormatUtils::VERSION_1:
return NO_FLAGS;
case BinaryDictionaryFormatUtils::VERSION_2:
return ByteArrayUtils::readUint16(binaryDictionaryInfo->getDictBuf(),
VERSION_2_MAGIC_NUMBER_SIZE + VERSION_2_DICTIONARY_VERSION_SIZE);

2
native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.h
View File

@ -82,8 +82,6 @@ class BinaryDictionaryHeaderReadingUtils {
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(BinaryDictionaryHeaderReadingUtils);
static const int FORMAT_VERSION_1_HEADER_SIZE;
static const int VERSION_2_MAGIC_NUMBER_SIZE;
static const int VERSION_2_DICTIONARY_VERSION_SIZE;
static const int VERSION_2_DICTIONARY_FLAG_SIZE;