nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Correctly read the header of APK-embedded dicts 

Bug: 13164518
Change-Id: I8768ad887af8b89ad9f29637f606c3c68629c7ca
main
Jean Chalard 2014-02-24 22:17:27 +09:00
parent b08a9e021c
commit 890b44e537
11 changed files with 43 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java
View File

@ -229,7 +229,7 @@ final public class BinaryDictionaryGetter {
private static boolean hackCanUseDictionaryFile(final Locale locale, final File f) { private static boolean hackCanUseDictionaryFile(final Locale locale, final File f) {
try { try {
// Read the version of the file // Read the version of the file
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(f); final DictDecoder dictDecoder = FormatSpec.getDictDecoder(f, 0, f.length());
final DictionaryHeader header = dictDecoder.readHeader(); final DictionaryHeader header = dictDecoder.readHeader();
final String version = header.mDictionaryOptions.mAttributes.get(VERSION_KEY); final String version = header.mDictionaryOptions.mAttributes.get(VERSION_KEY);

2
java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java
View File

@ -357,7 +357,7 @@ public final class BinaryDictDecoderUtils {
* @return true if it's a binary dictionary, false otherwise * @return true if it's a binary dictionary, false otherwise
*/ */
public static boolean isBinaryDictionary(final File file) { public static boolean isBinaryDictionary(final File file) {
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file); final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, 0, file.length());
if (dictDecoder == null) { if (dictDecoder == null) {
return false; return false;
} }

5
java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java
View File

@ -237,7 +237,7 @@ public final class BinaryDictIOUtils {
final File file, final long offset, final long length) final File file, final long offset, final long length)
throws FileNotFoundException, IOException, UnsupportedFormatException { throws FileNotFoundException, IOException, UnsupportedFormatException {
final byte[] buffer = new byte[HEADER_READING_BUFFER_SIZE]; final byte[] buffer = new byte[HEADER_READING_BUFFER_SIZE];
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, offset, length,
new DictDecoder.DictionaryBufferFactory() { new DictDecoder.DictionaryBufferFactory() {
@Override @Override
public DictBuffer getDictionaryBuffer(File file) public DictBuffer getDictionaryBuffer(File file)
@ -251,8 +251,7 @@ public final class BinaryDictIOUtils {
inStream.close(); inStream.close();
} }
} }
} });
);
if (dictDecoder == null) { if (dictDecoder == null) {
return null; return null;
} }

18
java/src/com/android/inputmethod/latin/makedict/FormatSpec.java
View File

@ -326,30 +326,34 @@ public final class FormatSpec {
* Returns new dictionary decoder. * Returns new dictionary decoder.
* *
* @param dictFile the dictionary file. * @param dictFile the dictionary file.
* @param offset the offset in the file.
* @param length the length of the file, in bytes.
* @param bufferType The type of buffer, as one of USE_* in DictDecoder. * @param bufferType The type of buffer, as one of USE_* in DictDecoder.
* @return new dictionary decoder if the dictionary file exists, otherwise null. * @return new dictionary decoder if the dictionary file exists, otherwise null.
*/ */
public static DictDecoder getDictDecoder(final File dictFile, final int bufferType) { public static DictDecoder getDictDecoder(final File dictFile, final long offset,
final long length, final int bufferType) {
if (dictFile.isDirectory()) { if (dictFile.isDirectory()) {
return new Ver4DictDecoder(dictFile, bufferType); return new Ver4DictDecoder(dictFile, bufferType);
} else if (dictFile.isFile()) { } else if (dictFile.isFile()) {
return new Ver2DictDecoder(dictFile, bufferType); return new Ver2DictDecoder(dictFile, offset, length, bufferType);
} }
return null; return null;
} }
public static DictDecoder getDictDecoder(final File dictFile, public static DictDecoder getDictDecoder(final File dictFile, final long offset,
final DictionaryBufferFactory factory) { final long length, final DictionaryBufferFactory factory) {
if (dictFile.isDirectory()) { if (dictFile.isDirectory()) {
return new Ver4DictDecoder(dictFile, factory); return new Ver4DictDecoder(dictFile, factory);
} else if (dictFile.isFile()) { } else if (dictFile.isFile()) {
return new Ver2DictDecoder(dictFile, factory); return new Ver2DictDecoder(dictFile, offset, length, factory);
} }
return null; return null;
} }
public static DictDecoder getDictDecoder(final File dictFile) { public static DictDecoder getDictDecoder(final File dictFile, final long offset,
return getDictDecoder(dictFile, DictDecoder.USE_READONLY_BYTEBUFFER); final long length) {
return getDictDecoder(dictFile, offset, length, DictDecoder.USE_READONLY_BYTEBUFFER);
} }
private FormatSpec() { private FormatSpec() {

18
java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java
View File

@ -116,13 +116,18 @@ public class Ver2DictDecoder extends AbstractDictDecoder {
} }
protected final File mDictionaryBinaryFile; protected final File mDictionaryBinaryFile;
protected final long mOffset;
protected final long mLength;
// TODO: Remove mBufferFactory and mDictBuffer from this class members because they are now // TODO: Remove mBufferFactory and mDictBuffer from this class members because they are now
// used only for testing. // used only for testing.
private final DictionaryBufferFactory mBufferFactory; private final DictionaryBufferFactory mBufferFactory;
protected DictBuffer mDictBuffer; protected DictBuffer mDictBuffer;
/* package */ Ver2DictDecoder(final File file, final int factoryFlag) { /* package */ Ver2DictDecoder(final File file, final long offset, final long length,
final int factoryFlag) {
mDictionaryBinaryFile = file; mDictionaryBinaryFile = file;
mOffset = offset;
mLength = length;
mDictBuffer = null; mDictBuffer = null;
if ((factoryFlag & MASK_DICTBUFFER) == USE_READONLY_BYTEBUFFER) { if ((factoryFlag & MASK_DICTBUFFER) == USE_READONLY_BYTEBUFFER) {
mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory(); mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory();
@ -135,8 +140,11 @@ public class Ver2DictDecoder extends AbstractDictDecoder {
} }
} }
/* package */ Ver2DictDecoder(final File file, final DictionaryBufferFactory factory) { /* package */ Ver2DictDecoder(final File file, final long offset, final long length,
final DictionaryBufferFactory factory) {
mDictionaryBinaryFile = file; mDictionaryBinaryFile = file;
mOffset = offset;
mLength = length;
mBufferFactory = factory; mBufferFactory = factory;
} }
@ -164,9 +172,9 @@ public class Ver2DictDecoder extends AbstractDictDecoder {
public DictionaryHeader readHeader() throws IOException, UnsupportedFormatException { public DictionaryHeader readHeader() throws IOException, UnsupportedFormatException {
// dictType is not being used in dicttool. Passing an empty string. // dictType is not being used in dicttool. Passing an empty string.
final BinaryDictionary binaryDictionary = new BinaryDictionary( final BinaryDictionary binaryDictionary = new BinaryDictionary(
mDictionaryBinaryFile.getAbsolutePath(), 0 /* offset */, mDictionaryBinaryFile.getAbsolutePath(), mOffset, mLength,
mDictionaryBinaryFile.length() /* length */, true /* useFullEditDistance */, true /* useFullEditDistance */, null /* locale */, "" /* dictType */,
null /* locale */, "" /* dictType */, false /* isUpdatable */); false /* isUpdatable */);
final DictionaryHeader header = binaryDictionary.getHeader(); final DictionaryHeader header = binaryDictionary.getHeader();
binaryDictionary.close(); binaryDictionary.close();
if (header == null) { if (header == null) {

2
tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
View File

@ -150,7 +150,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
binaryDictionary.flushWithGC(); binaryDictionary.flushWithGC();
binaryDictionary.close(); binaryDictionary.close();
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(dictFile); final DictDecoder dictDecoder = FormatSpec.getDictDecoder(dictFile, 0, dictFile.length());
try { try {
final FusionDictionary dict = final FusionDictionary dict =
dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */); dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */);

9
tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java
View File

@ -251,7 +251,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
FusionDictionary dict = null; FusionDictionary dict = null;
try { try {
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, bufferType); final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, 0, file.length(),
bufferType);
now = System.currentTimeMillis(); now = System.currentTimeMillis();
dict = dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */); dict = dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */);
diff = System.currentTimeMillis() - now; diff = System.currentTimeMillis() - now;
@ -413,7 +414,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
long now = -1, diff = -1; long now = -1, diff = -1;
try { try {
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, bufferType); final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, 0, file.length(),
bufferType);
now = System.currentTimeMillis(); now = System.currentTimeMillis();
dictDecoder.readUnigramsAndBigramsBinary(resultWords, resultFreqs, resultBigrams); dictDecoder.readUnigramsAndBigramsBinary(resultWords, resultFreqs, resultBigrams);
diff = System.currentTimeMillis() - now; diff = System.currentTimeMillis() - now;
@ -537,7 +539,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
addBigrams(dict, words, bigrams); addBigrams(dict, words, bigrams);
timeWritingDictToFile(file, dict, formatOptions); timeWritingDictToFile(file, dict, formatOptions);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, DictDecoder.USE_BYTEARRAY); final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, 0, file.length(),
DictDecoder.USE_BYTEARRAY);
try { try {
dictDecoder.openDictBuffer(); dictDecoder.openDictBuffer();
} catch (IOException e) { } catch (IOException e) {

6
tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoderTests.java
View File

@ -68,7 +68,8 @@ public class Ver2DictDecoderTests extends AndroidTestCase {
} }
assertNotNull(testFile); assertNotNull(testFile);
final Ver2DictDecoder dictDecoder = new Ver2DictDecoder(testFile, factory); final Ver2DictDecoder dictDecoder = new Ver2DictDecoder(testFile, 0, testFile.length(),
factory);
try { try {
dictDecoder.openDictBuffer(); dictDecoder.openDictBuffer();
} catch (Exception e) { } catch (Exception e) {
@ -110,7 +111,8 @@ public class Ver2DictDecoderTests extends AndroidTestCase {
Log.e(TAG, "IOException while the creating temporary file", e); Log.e(TAG, "IOException while the creating temporary file", e);
} }
final Ver2DictDecoder dictDecoder = new Ver2DictDecoder(testFile, factory); final Ver2DictDecoder dictDecoder = new Ver2DictDecoder(testFile, 0, testFile.length(),
factory);
// the default return value of getBuffer() must be null. // the default return value of getBuffer() must be null.
assertNull("the default return value of getBuffer() is not null", assertNull("the default return value of getBuffer() is not null",

2
tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java
View File

@ -192,7 +192,7 @@ public final class BinaryDictOffdeviceUtils {
new BufferedInputStream(new FileInputStream(decodedSpec.mFile))); new BufferedInputStream(new FileInputStream(decodedSpec.mFile)));
} else { } else {
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(decodedSpec.mFile, final DictDecoder dictDecoder = FormatSpec.getDictDecoder(decodedSpec.mFile,
DictDecoder.USE_BYTEARRAY); 0, decodedSpec.mFile.length(), DictDecoder.USE_BYTEARRAY);
if (report) { if (report) {
System.out.println("Format : Binary dictionary format"); System.out.println("Format : Binary dictionary format");
System.out.println("Packaging : " + decodedSpec.describeChain()); System.out.println("Packaging : " + decodedSpec.describeChain());

2
tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java
View File

@ -264,7 +264,7 @@ public class DictionaryMaker {
private static FusionDictionary readBinaryFile(final String binaryFilename) private static FusionDictionary readBinaryFile(final String binaryFilename)
throws FileNotFoundException, IOException, UnsupportedFormatException { throws FileNotFoundException, IOException, UnsupportedFormatException {
final File file = new File(binaryFilename); final File file = new File(binaryFilename);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file); final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, 0, file.length());
return dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */); return dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */);
} }

3
tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java
View File

@ -77,7 +77,8 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase {
assertEquals("Wrong decode spec", BinaryDictOffdeviceUtils.COMPRESSION, step); assertEquals("Wrong decode spec", BinaryDictOffdeviceUtils.COMPRESSION, step);
} }
assertEquals("Wrong decode spec", 3, decodeSpec.mDecoderSpec.size()); assertEquals("Wrong decode spec", 3, decodeSpec.mDecoderSpec.size());
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(decodeSpec.mFile); final DictDecoder dictDecoder = FormatSpec.getDictDecoder(decodeSpec.mFile, 0,
decodeSpec.mFile.length());
final FusionDictionary resultDict = final FusionDictionary resultDict =
dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */); dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */);
assertEquals("Wrong version attribute", VERSION, resultDict.mOptions.mAttributes.get( assertEquals("Wrong version attribute", VERSION, resultDict.mOptions.mAttributes.get(