nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Correctly read the header of APK-embedded dicts 

Bug: 13164518
Change-Id: I8768ad887af8b89ad9f29637f606c3c68629c7ca
main
Jean Chalard 2014-02-24 22:17:27 +09:00
parent b08a9e021c
commit 890b44e537
11 changed files with 43 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java
View File

@ -229,7 +229,7 @@ final public class BinaryDictionaryGetter {
private static boolean hackCanUseDictionaryFile(final Locale locale, final File f) {
try {
// Read the version of the file
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(f);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(f, 0, f.length());
final DictionaryHeader header = dictDecoder.readHeader();
final String version = header.mDictionaryOptions.mAttributes.get(VERSION_KEY);

2
java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java
View File

@ -357,7 +357,7 @@ public final class BinaryDictDecoderUtils {
* @return true if it's a binary dictionary, false otherwise
*/
public static boolean isBinaryDictionary(final File file) {
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, 0, file.length());
if (dictDecoder == null) {
return false;
}

5
java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java
View File

@ -237,7 +237,7 @@ public final class BinaryDictIOUtils {
final File file, final long offset, final long length)
throws FileNotFoundException, IOException, UnsupportedFormatException {
final byte[] buffer = new byte[HEADER_READING_BUFFER_SIZE];
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file,
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, offset, length,
new DictDecoder.DictionaryBufferFactory() {
@Override
public DictBuffer getDictionaryBuffer(File file)
@ -251,8 +251,7 @@ public final class BinaryDictIOUtils {
inStream.close();
}
}
}
);
});
if (dictDecoder == null) {
return null;
}

18
java/src/com/android/inputmethod/latin/makedict/FormatSpec.java
View File

@ -326,30 +326,34 @@ public final class FormatSpec {
* Returns new dictionary decoder.
*
* @param dictFile the dictionary file.
* @param offset the offset in the file.
* @param length the length of the file, in bytes.
* @param bufferType The type of buffer, as one of USE_* in DictDecoder.
* @return new dictionary decoder if the dictionary file exists, otherwise null.
*/
public static DictDecoder getDictDecoder(final File dictFile, final int bufferType) {
public static DictDecoder getDictDecoder(final File dictFile, final long offset,
final long length, final int bufferType) {
if (dictFile.isDirectory()) {
return new Ver4DictDecoder(dictFile, bufferType);
} else if (dictFile.isFile()) {
return new Ver2DictDecoder(dictFile, bufferType);
return new Ver2DictDecoder(dictFile, offset, length, bufferType);
}
return null;
}
public static DictDecoder getDictDecoder(final File dictFile,
final DictionaryBufferFactory factory) {
public static DictDecoder getDictDecoder(final File dictFile, final long offset,
final long length, final DictionaryBufferFactory factory) {
if (dictFile.isDirectory()) {
return new Ver4DictDecoder(dictFile, factory);
} else if (dictFile.isFile()) {
return new Ver2DictDecoder(dictFile, factory);
return new Ver2DictDecoder(dictFile, offset, length, factory);
}
return null;
}
public static DictDecoder getDictDecoder(final File dictFile) {
return getDictDecoder(dictFile, DictDecoder.USE_READONLY_BYTEBUFFER);
public static DictDecoder getDictDecoder(final File dictFile, final long offset,
final long length) {
return getDictDecoder(dictFile, offset, length, DictDecoder.USE_READONLY_BYTEBUFFER);
}
private FormatSpec() {

18
java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java
View File

@ -116,13 +116,18 @@ public class Ver2DictDecoder extends AbstractDictDecoder {
}
protected final File mDictionaryBinaryFile;
protected final long mOffset;
protected final long mLength;
// TODO: Remove mBufferFactory and mDictBuffer from this class members because they are now
// used only for testing.
private final DictionaryBufferFactory mBufferFactory;
protected DictBuffer mDictBuffer;
/* package */ Ver2DictDecoder(final File file, final int factoryFlag) {
/* package */ Ver2DictDecoder(final File file, final long offset, final long length,
final int factoryFlag) {
mDictionaryBinaryFile = file;
mOffset = offset;
mLength = length;
mDictBuffer = null;
if ((factoryFlag & MASK_DICTBUFFER) == USE_READONLY_BYTEBUFFER) {
mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory();
@ -135,8 +140,11 @@ public class Ver2DictDecoder extends AbstractDictDecoder {
}
}
/* package */ Ver2DictDecoder(final File file, final DictionaryBufferFactory factory) {
/* package */ Ver2DictDecoder(final File file, final long offset, final long length,
final DictionaryBufferFactory factory) {
mDictionaryBinaryFile = file;
mOffset = offset;
mLength = length;
mBufferFactory = factory;
}
@ -164,9 +172,9 @@ public class Ver2DictDecoder extends AbstractDictDecoder {
public DictionaryHeader readHeader() throws IOException, UnsupportedFormatException {
// dictType is not being used in dicttool. Passing an empty string.
final BinaryDictionary binaryDictionary = new BinaryDictionary(
mDictionaryBinaryFile.getAbsolutePath(), 0 /* offset */,
mDictionaryBinaryFile.length() /* length */, true /* useFullEditDistance */,
null /* locale */, "" /* dictType */, false /* isUpdatable */);
mDictionaryBinaryFile.getAbsolutePath(), mOffset, mLength,
true /* useFullEditDistance */, null /* locale */, "" /* dictType */,
false /* isUpdatable */);
final DictionaryHeader header = binaryDictionary.getHeader();
binaryDictionary.close();
if (header == null) {

2
tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
View File

@ -150,7 +150,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
binaryDictionary.flushWithGC();
binaryDictionary.close();
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(dictFile);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(dictFile, 0, dictFile.length());
try {
final FusionDictionary dict =
dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */);

9
tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java
View File

@ -251,7 +251,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
FusionDictionary dict = null;
try {
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, bufferType);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, 0, file.length(),
bufferType);
now = System.currentTimeMillis();
dict = dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */);
diff = System.currentTimeMillis() - now;
@ -413,7 +414,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
long now = -1, diff = -1;
try {
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, bufferType);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, 0, file.length(),
bufferType);
now = System.currentTimeMillis();
dictDecoder.readUnigramsAndBigramsBinary(resultWords, resultFreqs, resultBigrams);
diff = System.currentTimeMillis() - now;
@ -537,7 +539,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
addBigrams(dict, words, bigrams);
timeWritingDictToFile(file, dict, formatOptions);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, DictDecoder.USE_BYTEARRAY);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, 0, file.length(),
DictDecoder.USE_BYTEARRAY);
try {
dictDecoder.openDictBuffer();
} catch (IOException e) {

6
tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoderTests.java
View File

@ -68,7 +68,8 @@ public class Ver2DictDecoderTests extends AndroidTestCase {
}
assertNotNull(testFile);
final Ver2DictDecoder dictDecoder = new Ver2DictDecoder(testFile, factory);
final Ver2DictDecoder dictDecoder = new Ver2DictDecoder(testFile, 0, testFile.length(),
factory);
try {
dictDecoder.openDictBuffer();
} catch (Exception e) {
@ -110,7 +111,8 @@ public class Ver2DictDecoderTests extends AndroidTestCase {
Log.e(TAG, "IOException while the creating temporary file", e);
}
final Ver2DictDecoder dictDecoder = new Ver2DictDecoder(testFile, factory);
final Ver2DictDecoder dictDecoder = new Ver2DictDecoder(testFile, 0, testFile.length(),
factory);
// the default return value of getBuffer() must be null.
assertNull("the default return value of getBuffer() is not null",

2
tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java
View File

@ -192,7 +192,7 @@ public final class BinaryDictOffdeviceUtils {
new BufferedInputStream(new FileInputStream(decodedSpec.mFile)));
} else {
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(decodedSpec.mFile,
DictDecoder.USE_BYTEARRAY);
0, decodedSpec.mFile.length(), DictDecoder.USE_BYTEARRAY);
if (report) {
System.out.println("Format : Binary dictionary format");
System.out.println("Packaging : " + decodedSpec.describeChain());

2
tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java
View File

@ -264,7 +264,7 @@ public class DictionaryMaker {
private static FusionDictionary readBinaryFile(final String binaryFilename)
throws FileNotFoundException, IOException, UnsupportedFormatException {
final File file = new File(binaryFilename);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, 0, file.length());
return dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */);
}

3
tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java
View File

@ -77,7 +77,8 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase {
assertEquals("Wrong decode spec", BinaryDictOffdeviceUtils.COMPRESSION, step);
}
assertEquals("Wrong decode spec", 3, decodeSpec.mDecoderSpec.size());
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(decodeSpec.mFile);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(decodeSpec.mFile, 0,
decodeSpec.mFile.length());
final FusionDictionary resultDict =
dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */);
assertEquals("Wrong version attribute", VERSION, resultDict.mOptions.mAttributes.get(