From 890b44e5376413adc73025e046072bcce3e119c5 Mon Sep 17 00:00:00 2001 From: Jean Chalard Date: Mon, 24 Feb 2014 22:17:27 +0900 Subject: [PATCH] Correctly read the header of APK-embedded dicts Bug: 13164518 Change-Id: I8768ad887af8b89ad9f29637f606c3c68629c7ca --- .../latin/BinaryDictionaryGetter.java | 2 +- .../latin/makedict/BinaryDictDecoderUtils.java | 2 +- .../latin/makedict/BinaryDictIOUtils.java | 5 ++--- .../inputmethod/latin/makedict/FormatSpec.java | 18 +++++++++++------- .../latin/makedict/Ver2DictDecoder.java | 18 +++++++++++++----- .../latin/BinaryDictionaryDecayingTests.java | 2 +- .../BinaryDictDecoderEncoderTests.java | 9 ++++++--- .../latin/makedict/Ver2DictDecoderTests.java | 6 ++++-- .../dicttool/BinaryDictOffdeviceUtils.java | 2 +- .../latin/dicttool/DictionaryMaker.java | 2 +- .../BinaryDictOffdeviceUtilsTests.java | 3 ++- 11 files changed, 43 insertions(+), 26 deletions(-) diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java b/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java index a7008379f..acbd919cd 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java @@ -229,7 +229,7 @@ final public class BinaryDictionaryGetter { private static boolean hackCanUseDictionaryFile(final Locale locale, final File f) { try { // Read the version of the file - final DictDecoder dictDecoder = FormatSpec.getDictDecoder(f); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(f, 0, f.length()); final DictionaryHeader header = dictDecoder.readHeader(); final String version = header.mDictionaryOptions.mAttributes.get(VERSION_KEY); diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java index b534ebeff..25e1bcd25 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java @@ -357,7 +357,7 @@ public final class BinaryDictDecoderUtils { * @return true if it's a binary dictionary, false otherwise */ public static boolean isBinaryDictionary(final File file) { - final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, 0, file.length()); if (dictDecoder == null) { return false; } diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java index 989ca4b2f..90e7400fb 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java @@ -237,7 +237,7 @@ public final class BinaryDictIOUtils { final File file, final long offset, final long length) throws FileNotFoundException, IOException, UnsupportedFormatException { final byte[] buffer = new byte[HEADER_READING_BUFFER_SIZE]; - final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, offset, length, new DictDecoder.DictionaryBufferFactory() { @Override public DictBuffer getDictionaryBuffer(File file) @@ -251,8 +251,7 @@ public final class BinaryDictIOUtils { inStream.close(); } } - } - ); + }); if (dictDecoder == null) { return null; } diff --git a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java index c7635eff9..9abecbfec 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java +++ b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java @@ -326,30 +326,34 @@ public final class FormatSpec { * Returns new dictionary decoder. * * @param dictFile the dictionary file. + * @param offset the offset in the file. + * @param length the length of the file, in bytes. * @param bufferType The type of buffer, as one of USE_* in DictDecoder. * @return new dictionary decoder if the dictionary file exists, otherwise null. */ - public static DictDecoder getDictDecoder(final File dictFile, final int bufferType) { + public static DictDecoder getDictDecoder(final File dictFile, final long offset, + final long length, final int bufferType) { if (dictFile.isDirectory()) { return new Ver4DictDecoder(dictFile, bufferType); } else if (dictFile.isFile()) { - return new Ver2DictDecoder(dictFile, bufferType); + return new Ver2DictDecoder(dictFile, offset, length, bufferType); } return null; } - public static DictDecoder getDictDecoder(final File dictFile, - final DictionaryBufferFactory factory) { + public static DictDecoder getDictDecoder(final File dictFile, final long offset, + final long length, final DictionaryBufferFactory factory) { if (dictFile.isDirectory()) { return new Ver4DictDecoder(dictFile, factory); } else if (dictFile.isFile()) { - return new Ver2DictDecoder(dictFile, factory); + return new Ver2DictDecoder(dictFile, offset, length, factory); } return null; } - public static DictDecoder getDictDecoder(final File dictFile) { - return getDictDecoder(dictFile, DictDecoder.USE_READONLY_BYTEBUFFER); + public static DictDecoder getDictDecoder(final File dictFile, final long offset, + final long length) { + return getDictDecoder(dictFile, offset, length, DictDecoder.USE_READONLY_BYTEBUFFER); } private FormatSpec() { diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java index bf776cfc5..ae1e443c5 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java @@ -116,13 +116,18 @@ public class Ver2DictDecoder extends AbstractDictDecoder { } protected final File mDictionaryBinaryFile; + protected final long mOffset; + protected final long mLength; // TODO: Remove mBufferFactory and mDictBuffer from this class members because they are now // used only for testing. private final DictionaryBufferFactory mBufferFactory; protected DictBuffer mDictBuffer; - /* package */ Ver2DictDecoder(final File file, final int factoryFlag) { + /* package */ Ver2DictDecoder(final File file, final long offset, final long length, + final int factoryFlag) { mDictionaryBinaryFile = file; + mOffset = offset; + mLength = length; mDictBuffer = null; if ((factoryFlag & MASK_DICTBUFFER) == USE_READONLY_BYTEBUFFER) { mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory(); @@ -135,8 +140,11 @@ public class Ver2DictDecoder extends AbstractDictDecoder { } } - /* package */ Ver2DictDecoder(final File file, final DictionaryBufferFactory factory) { + /* package */ Ver2DictDecoder(final File file, final long offset, final long length, + final DictionaryBufferFactory factory) { mDictionaryBinaryFile = file; + mOffset = offset; + mLength = length; mBufferFactory = factory; } @@ -164,9 +172,9 @@ public class Ver2DictDecoder extends AbstractDictDecoder { public DictionaryHeader readHeader() throws IOException, UnsupportedFormatException { // dictType is not being used in dicttool. Passing an empty string. final BinaryDictionary binaryDictionary = new BinaryDictionary( - mDictionaryBinaryFile.getAbsolutePath(), 0 /* offset */, - mDictionaryBinaryFile.length() /* length */, true /* useFullEditDistance */, - null /* locale */, "" /* dictType */, false /* isUpdatable */); + mDictionaryBinaryFile.getAbsolutePath(), mOffset, mLength, + true /* useFullEditDistance */, null /* locale */, "" /* dictType */, + false /* isUpdatable */); final DictionaryHeader header = binaryDictionary.getHeader(); binaryDictionary.close(); if (header == null) { diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java index f4b16a7e1..fa5123665 100644 --- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java @@ -150,7 +150,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { binaryDictionary.flushWithGC(); binaryDictionary.close(); - final DictDecoder dictDecoder = FormatSpec.getDictDecoder(dictFile); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(dictFile, 0, dictFile.length()); try { final FusionDictionary dict = dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */); diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java index e21e340c2..0ee0fb577 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java @@ -251,7 +251,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { FusionDictionary dict = null; try { - final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, bufferType); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, 0, file.length(), + bufferType); now = System.currentTimeMillis(); dict = dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */); diff = System.currentTimeMillis() - now; @@ -413,7 +414,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { long now = -1, diff = -1; try { - final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, bufferType); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, 0, file.length(), + bufferType); now = System.currentTimeMillis(); dictDecoder.readUnigramsAndBigramsBinary(resultWords, resultFreqs, resultBigrams); diff = System.currentTimeMillis() - now; @@ -537,7 +539,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { addBigrams(dict, words, bigrams); timeWritingDictToFile(file, dict, formatOptions); - final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, DictDecoder.USE_BYTEARRAY); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, 0, file.length(), + DictDecoder.USE_BYTEARRAY); try { dictDecoder.openDictBuffer(); } catch (IOException e) { diff --git a/tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoderTests.java b/tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoderTests.java index a85753e6b..9dc2b1058 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoderTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoderTests.java @@ -68,7 +68,8 @@ public class Ver2DictDecoderTests extends AndroidTestCase { } assertNotNull(testFile); - final Ver2DictDecoder dictDecoder = new Ver2DictDecoder(testFile, factory); + final Ver2DictDecoder dictDecoder = new Ver2DictDecoder(testFile, 0, testFile.length(), + factory); try { dictDecoder.openDictBuffer(); } catch (Exception e) { @@ -110,7 +111,8 @@ public class Ver2DictDecoderTests extends AndroidTestCase { Log.e(TAG, "IOException while the creating temporary file", e); } - final Ver2DictDecoder dictDecoder = new Ver2DictDecoder(testFile, factory); + final Ver2DictDecoder dictDecoder = new Ver2DictDecoder(testFile, 0, testFile.length(), + factory); // the default return value of getBuffer() must be null. assertNull("the default return value of getBuffer() is not null", diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java index d1df81b52..e31ac2ab6 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java @@ -192,7 +192,7 @@ public final class BinaryDictOffdeviceUtils { new BufferedInputStream(new FileInputStream(decodedSpec.mFile))); } else { final DictDecoder dictDecoder = FormatSpec.getDictDecoder(decodedSpec.mFile, - DictDecoder.USE_BYTEARRAY); + 0, decodedSpec.mFile.length(), DictDecoder.USE_BYTEARRAY); if (report) { System.out.println("Format : Binary dictionary format"); System.out.println("Packaging : " + decodedSpec.describeChain()); diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java index 80d71fc64..68d785044 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java @@ -264,7 +264,7 @@ public class DictionaryMaker { private static FusionDictionary readBinaryFile(final String binaryFilename) throws FileNotFoundException, IOException, UnsupportedFormatException { final File file = new File(binaryFilename); - final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, 0, file.length()); return dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */); } diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java index 7a4f6f7c5..faf00b4a5 100644 --- a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java +++ b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java @@ -77,7 +77,8 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase { assertEquals("Wrong decode spec", BinaryDictOffdeviceUtils.COMPRESSION, step); } assertEquals("Wrong decode spec", 3, decodeSpec.mDecoderSpec.size()); - final DictDecoder dictDecoder = FormatSpec.getDictDecoder(decodeSpec.mFile); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(decodeSpec.mFile, 0, + decodeSpec.mFile.length()); final FusionDictionary resultDict = dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */); assertEquals("Wrong version attribute", VERSION, resultDict.mOptions.mAttributes.get(