nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge "Correctly read the header of APK-embedded dicts"

main
Jean Chalard 2014-02-24 13:57:46 +00:00 committed by Android (Google) Code Review
parent 64ace089a8 890b44e537
commit 6e224a236a
11 changed files with 43 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java
View File

@ -229,7 +229,7 @@ final public class BinaryDictionaryGetter {
private static boolean hackCanUseDictionaryFile(final Locale locale, final File f) {
try {
// Read the version of the file
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(f);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(f, 0, f.length());
final DictionaryHeader header = dictDecoder.readHeader();
final String version = header.mDictionaryOptions.mAttributes.get(VERSION_KEY);

2
java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java
View File

@ -357,7 +357,7 @@ public final class BinaryDictDecoderUtils {
* @return true if it's a binary dictionary, false otherwise
*/
public static boolean isBinaryDictionary(final File file) {
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, 0, file.length());
if (dictDecoder == null) {
return false;
}

5
java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java
View File

@ -237,7 +237,7 @@ public final class BinaryDictIOUtils {
final File file, final long offset, final long length)
throws FileNotFoundException, IOException, UnsupportedFormatException {
final byte[] buffer = new byte[HEADER_READING_BUFFER_SIZE];
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file,
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, offset, length,
new DictDecoder.DictionaryBufferFactory() {
@Override
public DictBuffer getDictionaryBuffer(File file)
@ -251,8 +251,7 @@ public final class BinaryDictIOUtils {
inStream.close();
}
}
}
);
});
if (dictDecoder == null) {
return null;
}

18
java/src/com/android/inputmethod/latin/makedict/FormatSpec.java
View File

@ -326,30 +326,34 @@ public final class FormatSpec {
* Returns new dictionary decoder.
*
* @param dictFile the dictionary file.
* @param offset the offset in the file.
* @param length the length of the file, in bytes.
* @param bufferType The type of buffer, as one of USE_* in DictDecoder.
* @return new dictionary decoder if the dictionary file exists, otherwise null.
*/
public static DictDecoder getDictDecoder(final File dictFile, final int bufferType) {
public static DictDecoder getDictDecoder(final File dictFile, final long offset,
final long length, final int bufferType) {
if (dictFile.isDirectory()) {
return new Ver4DictDecoder(dictFile, bufferType);
} else if (dictFile.isFile()) {
return new Ver2DictDecoder(dictFile, bufferType);
return new Ver2DictDecoder(dictFile, offset, length, bufferType);
}
return null;
}
public static DictDecoder getDictDecoder(final File dictFile,
final DictionaryBufferFactory factory) {
public static DictDecoder getDictDecoder(final File dictFile, final long offset,
final long length, final DictionaryBufferFactory factory) {
if (dictFile.isDirectory()) {
return new Ver4DictDecoder(dictFile, factory);
} else if (dictFile.isFile()) {
return new Ver2DictDecoder(dictFile, factory);
return new Ver2DictDecoder(dictFile, offset, length, factory);
}
return null;
}
public static DictDecoder getDictDecoder(final File dictFile) {
return getDictDecoder(dictFile, DictDecoder.USE_READONLY_BYTEBUFFER);
public static DictDecoder getDictDecoder(final File dictFile, final long offset,
final long length) {
return getDictDecoder(dictFile, offset, length, DictDecoder.USE_READONLY_BYTEBUFFER);
}
private FormatSpec() {

18
java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java
View File

@ -116,13 +116,18 @@ public class Ver2DictDecoder extends AbstractDictDecoder {
}
protected final File mDictionaryBinaryFile;
protected final long mOffset;
protected final long mLength;
// TODO: Remove mBufferFactory and mDictBuffer from this class members because they are now
// used only for testing.
private final DictionaryBufferFactory mBufferFactory;
protected DictBuffer mDictBuffer;
/* package */ Ver2DictDecoder(final File file, final int factoryFlag) {
/* package */ Ver2DictDecoder(final File file, final long offset, final long length,
final int factoryFlag) {
mDictionaryBinaryFile = file;
mOffset = offset;
mLength = length;
mDictBuffer = null;
if ((factoryFlag & MASK_DICTBUFFER) == USE_READONLY_BYTEBUFFER) {
mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory();
@ -135,8 +140,11 @@ public class Ver2DictDecoder extends AbstractDictDecoder {
}
}
/* package */ Ver2DictDecoder(final File file, final DictionaryBufferFactory factory) {
/* package */ Ver2DictDecoder(final File file, final long offset, final long length,
final DictionaryBufferFactory factory) {
mDictionaryBinaryFile = file;
mOffset = offset;
mLength = length;
mBufferFactory = factory;
}
@ -164,9 +172,9 @@ public class Ver2DictDecoder extends AbstractDictDecoder {
public DictionaryHeader readHeader() throws IOException, UnsupportedFormatException {
// dictType is not being used in dicttool. Passing an empty string.
final BinaryDictionary binaryDictionary = new BinaryDictionary(
mDictionaryBinaryFile.getAbsolutePath(), 0 /* offset */,
mDictionaryBinaryFile.length() /* length */, true /* useFullEditDistance */,
null /* locale */, "" /* dictType */, false /* isUpdatable */);
mDictionaryBinaryFile.getAbsolutePath(), mOffset, mLength,
true /* useFullEditDistance */, null /* locale */, "" /* dictType */,
false /* isUpdatable */);
final DictionaryHeader header = binaryDictionary.getHeader();
binaryDictionary.close();
if (header == null) {

2
tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
View File

@ -150,7 +150,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
binaryDictionary.flushWithGC();
binaryDictionary.close();
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(dictFile);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(dictFile, 0, dictFile.length());
try {
final FusionDictionary dict =
dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */);

9
tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java
View File

@ -251,7 +251,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
FusionDictionary dict = null;
try {
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, bufferType);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, 0, file.length(),
bufferType);
now = System.currentTimeMillis();
dict = dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */);
diff = System.currentTimeMillis() - now;
@ -413,7 +414,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
long now = -1, diff = -1;
try {
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, bufferType);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, 0, file.length(),
bufferType);
now = System.currentTimeMillis();
dictDecoder.readUnigramsAndBigramsBinary(resultWords, resultFreqs, resultBigrams);
diff = System.currentTimeMillis() - now;
@ -537,7 +539,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
addBigrams(dict, words, bigrams);
timeWritingDictToFile(file, dict, formatOptions);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, DictDecoder.USE_BYTEARRAY);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, 0, file.length(),
DictDecoder.USE_BYTEARRAY);
try {
dictDecoder.openDictBuffer();
} catch (IOException e) {

6
tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoderTests.java
View File

@ -68,7 +68,8 @@ public class Ver2DictDecoderTests extends AndroidTestCase {
}
assertNotNull(testFile);
final Ver2DictDecoder dictDecoder = new Ver2DictDecoder(testFile, factory);
final Ver2DictDecoder dictDecoder = new Ver2DictDecoder(testFile, 0, testFile.length(),
factory);
try {
dictDecoder.openDictBuffer();
} catch (Exception e) {
@ -110,7 +111,8 @@ public class Ver2DictDecoderTests extends AndroidTestCase {
Log.e(TAG, "IOException while the creating temporary file", e);
}
final Ver2DictDecoder dictDecoder = new Ver2DictDecoder(testFile, factory);
final Ver2DictDecoder dictDecoder = new Ver2DictDecoder(testFile, 0, testFile.length(),
factory);
// the default return value of getBuffer() must be null.
assertNull("the default return value of getBuffer() is not null",

2
tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java
View File

@ -192,7 +192,7 @@ public final class BinaryDictOffdeviceUtils {
new BufferedInputStream(new FileInputStream(decodedSpec.mFile)));
} else {
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(decodedSpec.mFile,
DictDecoder.USE_BYTEARRAY);
0, decodedSpec.mFile.length(), DictDecoder.USE_BYTEARRAY);
if (report) {
System.out.println("Format : Binary dictionary format");
System.out.println("Packaging : " + decodedSpec.describeChain());

2
tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java
View File

@ -264,7 +264,7 @@ public class DictionaryMaker {
private static FusionDictionary readBinaryFile(final String binaryFilename)
throws FileNotFoundException, IOException, UnsupportedFormatException {
final File file = new File(binaryFilename);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, 0, file.length());
return dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */);
}

3
tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java
View File

@ -77,7 +77,8 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase {
assertEquals("Wrong decode spec", BinaryDictOffdeviceUtils.COMPRESSION, step);
}
assertEquals("Wrong decode spec", 3, decodeSpec.mDecoderSpec.size());
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(decodeSpec.mFile);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(decodeSpec.mFile, 0,
decodeSpec.mFile.length());
final FusionDictionary resultDict =
dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */);
assertEquals("Wrong version attribute", VERSION, resultDict.mOptions.mAttributes.get(