nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge "Correctly read the header of APK-embedded dicts"

main
Jean Chalard 2014-02-24 13:57:46 +00:00 committed by Android (Google) Code Review
parent 64ace089a8 890b44e537
commit 6e224a236a
11 changed files with 43 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java
View File

@ -229,7 +229,7 @@ final public class BinaryDictionaryGetter {
private static boolean hackCanUseDictionaryFile(final Locale locale, final File f) { private static boolean hackCanUseDictionaryFile(final Locale locale, final File f) {
try { try {
// Read the version of the file // Read the version of the file
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(f); final DictDecoder dictDecoder = FormatSpec.getDictDecoder(f, 0, f.length());
final DictionaryHeader header = dictDecoder.readHeader(); final DictionaryHeader header = dictDecoder.readHeader();
final String version = header.mDictionaryOptions.mAttributes.get(VERSION_KEY); final String version = header.mDictionaryOptions.mAttributes.get(VERSION_KEY);

2
java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java
View File

@ -357,7 +357,7 @@ public final class BinaryDictDecoderUtils {
* @return true if it's a binary dictionary, false otherwise * @return true if it's a binary dictionary, false otherwise
*/ */
public static boolean isBinaryDictionary(final File file) { public static boolean isBinaryDictionary(final File file) {
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file); final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, 0, file.length());
if (dictDecoder == null) { if (dictDecoder == null) {
return false; return false;
} }

5
java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java
View File

@ -237,7 +237,7 @@ public final class BinaryDictIOUtils {
final File file, final long offset, final long length) final File file, final long offset, final long length)
throws FileNotFoundException, IOException, UnsupportedFormatException { throws FileNotFoundException, IOException, UnsupportedFormatException {
final byte[] buffer = new byte[HEADER_READING_BUFFER_SIZE]; final byte[] buffer = new byte[HEADER_READING_BUFFER_SIZE];
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, offset, length,
new DictDecoder.DictionaryBufferFactory() { new DictDecoder.DictionaryBufferFactory() {
@Override @Override
public DictBuffer getDictionaryBuffer(File file) public DictBuffer getDictionaryBuffer(File file)
@ -251,8 +251,7 @@ public final class BinaryDictIOUtils {
inStream.close(); inStream.close();
} }
} }
} });
);
if (dictDecoder == null) { if (dictDecoder == null) {
return null; return null;
} }

18
java/src/com/android/inputmethod/latin/makedict/FormatSpec.java
View File

@ -326,30 +326,34 @@ public final class FormatSpec {
* Returns new dictionary decoder. * Returns new dictionary decoder.
* *
* @param dictFile the dictionary file. * @param dictFile the dictionary file.
* @param offset the offset in the file.
* @param length the length of the file, in bytes.
* @param bufferType The type of buffer, as one of USE_* in DictDecoder. * @param bufferType The type of buffer, as one of USE_* in DictDecoder.
* @return new dictionary decoder if the dictionary file exists, otherwise null. * @return new dictionary decoder if the dictionary file exists, otherwise null.
*/ */
public static DictDecoder getDictDecoder(final File dictFile, final int bufferType) { public static DictDecoder getDictDecoder(final File dictFile, final long offset,
final long length, final int bufferType) {
if (dictFile.isDirectory()) { if (dictFile.isDirectory()) {
return new Ver4DictDecoder(dictFile, bufferType); return new Ver4DictDecoder(dictFile, bufferType);
} else if (dictFile.isFile()) { } else if (dictFile.isFile()) {
return new Ver2DictDecoder(dictFile, bufferType); return new Ver2DictDecoder(dictFile, offset, length, bufferType);
} }
return null; return null;
} }
public static DictDecoder getDictDecoder(final File dictFile, public static DictDecoder getDictDecoder(final File dictFile, final long offset,
final DictionaryBufferFactory factory) { final long length, final DictionaryBufferFactory factory) {
if (dictFile.isDirectory()) { if (dictFile.isDirectory()) {
return new Ver4DictDecoder(dictFile, factory); return new Ver4DictDecoder(dictFile, factory);
} else if (dictFile.isFile()) { } else if (dictFile.isFile()) {
return new Ver2DictDecoder(dictFile, factory); return new Ver2DictDecoder(dictFile, offset, length, factory);
} }
return null; return null;
} }
public static DictDecoder getDictDecoder(final File dictFile) { public static DictDecoder getDictDecoder(final File dictFile, final long offset,
return getDictDecoder(dictFile, DictDecoder.USE_READONLY_BYTEBUFFER); final long length) {
return getDictDecoder(dictFile, offset, length, DictDecoder.USE_READONLY_BYTEBUFFER);
} }
private FormatSpec() { private FormatSpec() {

18
java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java
View File

@ -116,13 +116,18 @@ public class Ver2DictDecoder extends AbstractDictDecoder {
} }
protected final File mDictionaryBinaryFile; protected final File mDictionaryBinaryFile;
protected final long mOffset;
protected final long mLength;
// TODO: Remove mBufferFactory and mDictBuffer from this class members because they are now // TODO: Remove mBufferFactory and mDictBuffer from this class members because they are now
// used only for testing. // used only for testing.
private final DictionaryBufferFactory mBufferFactory; private final DictionaryBufferFactory mBufferFactory;
protected DictBuffer mDictBuffer; protected DictBuffer mDictBuffer;
/* package */ Ver2DictDecoder(final File file, final int factoryFlag) { /* package */ Ver2DictDecoder(final File file, final long offset, final long length,
final int factoryFlag) {
mDictionaryBinaryFile = file; mDictionaryBinaryFile = file;
mOffset = offset;
mLength = length;
mDictBuffer = null; mDictBuffer = null;
if ((factoryFlag & MASK_DICTBUFFER) == USE_READONLY_BYTEBUFFER) { if ((factoryFlag & MASK_DICTBUFFER) == USE_READONLY_BYTEBUFFER) {
mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory(); mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory();
@ -135,8 +140,11 @@ public class Ver2DictDecoder extends AbstractDictDecoder {
} }
} }
/* package */ Ver2DictDecoder(final File file, final DictionaryBufferFactory factory) { /* package */ Ver2DictDecoder(final File file, final long offset, final long length,
final DictionaryBufferFactory factory) {
mDictionaryBinaryFile = file; mDictionaryBinaryFile = file;
mOffset = offset;
mLength = length;
mBufferFactory = factory; mBufferFactory = factory;
} }
@ -164,9 +172,9 @@ public class Ver2DictDecoder extends AbstractDictDecoder {
public DictionaryHeader readHeader() throws IOException, UnsupportedFormatException { public DictionaryHeader readHeader() throws IOException, UnsupportedFormatException {
// dictType is not being used in dicttool. Passing an empty string. // dictType is not being used in dicttool. Passing an empty string.
final BinaryDictionary binaryDictionary = new BinaryDictionary( final BinaryDictionary binaryDictionary = new BinaryDictionary(
mDictionaryBinaryFile.getAbsolutePath(), 0 /* offset */, mDictionaryBinaryFile.getAbsolutePath(), mOffset, mLength,
mDictionaryBinaryFile.length() /* length */, true /* useFullEditDistance */, true /* useFullEditDistance */, null /* locale */, "" /* dictType */,
null /* locale */, "" /* dictType */, false /* isUpdatable */); false /* isUpdatable */);
final DictionaryHeader header = binaryDictionary.getHeader(); final DictionaryHeader header = binaryDictionary.getHeader();
binaryDictionary.close(); binaryDictionary.close();
if (header == null) { if (header == null) {

2
tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
View File

@ -150,7 +150,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
binaryDictionary.flushWithGC(); binaryDictionary.flushWithGC();
binaryDictionary.close(); binaryDictionary.close();
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(dictFile); final DictDecoder dictDecoder = FormatSpec.getDictDecoder(dictFile, 0, dictFile.length());
try { try {
final FusionDictionary dict = final FusionDictionary dict =
dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */); dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */);

9
tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java
View File

@ -251,7 +251,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
FusionDictionary dict = null; FusionDictionary dict = null;
try { try {
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, bufferType); final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, 0, file.length(),
bufferType);
now = System.currentTimeMillis(); now = System.currentTimeMillis();
dict = dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */); dict = dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */);
diff = System.currentTimeMillis() - now; diff = System.currentTimeMillis() - now;
@ -413,7 +414,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
long now = -1, diff = -1; long now = -1, diff = -1;
try { try {
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, bufferType); final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, 0, file.length(),
bufferType);
now = System.currentTimeMillis(); now = System.currentTimeMillis();
dictDecoder.readUnigramsAndBigramsBinary(resultWords, resultFreqs, resultBigrams); dictDecoder.readUnigramsAndBigramsBinary(resultWords, resultFreqs, resultBigrams);
diff = System.currentTimeMillis() - now; diff = System.currentTimeMillis() - now;
@ -537,7 +539,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
addBigrams(dict, words, bigrams); addBigrams(dict, words, bigrams);
timeWritingDictToFile(file, dict, formatOptions); timeWritingDictToFile(file, dict, formatOptions);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, DictDecoder.USE_BYTEARRAY); final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, 0, file.length(),
DictDecoder.USE_BYTEARRAY);
try { try {
dictDecoder.openDictBuffer(); dictDecoder.openDictBuffer();
} catch (IOException e) { } catch (IOException e) {

6
tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoderTests.java
View File

@ -68,7 +68,8 @@ public class Ver2DictDecoderTests extends AndroidTestCase {
} }
assertNotNull(testFile); assertNotNull(testFile);
final Ver2DictDecoder dictDecoder = new Ver2DictDecoder(testFile, factory); final Ver2DictDecoder dictDecoder = new Ver2DictDecoder(testFile, 0, testFile.length(),
factory);
try { try {
dictDecoder.openDictBuffer(); dictDecoder.openDictBuffer();
} catch (Exception e) { } catch (Exception e) {
@ -110,7 +111,8 @@ public class Ver2DictDecoderTests extends AndroidTestCase {
Log.e(TAG, "IOException while the creating temporary file", e); Log.e(TAG, "IOException while the creating temporary file", e);
} }
final Ver2DictDecoder dictDecoder = new Ver2DictDecoder(testFile, factory); final Ver2DictDecoder dictDecoder = new Ver2DictDecoder(testFile, 0, testFile.length(),
factory);
// the default return value of getBuffer() must be null. // the default return value of getBuffer() must be null.
assertNull("the default return value of getBuffer() is not null", assertNull("the default return value of getBuffer() is not null",

2
tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java
View File

@ -192,7 +192,7 @@ public final class BinaryDictOffdeviceUtils {
new BufferedInputStream(new FileInputStream(decodedSpec.mFile))); new BufferedInputStream(new FileInputStream(decodedSpec.mFile)));
} else { } else {
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(decodedSpec.mFile, final DictDecoder dictDecoder = FormatSpec.getDictDecoder(decodedSpec.mFile,
DictDecoder.USE_BYTEARRAY); 0, decodedSpec.mFile.length(), DictDecoder.USE_BYTEARRAY);
if (report) { if (report) {
System.out.println("Format : Binary dictionary format"); System.out.println("Format : Binary dictionary format");
System.out.println("Packaging : " + decodedSpec.describeChain()); System.out.println("Packaging : " + decodedSpec.describeChain());

2
tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java
View File

@ -264,7 +264,7 @@ public class DictionaryMaker {
private static FusionDictionary readBinaryFile(final String binaryFilename) private static FusionDictionary readBinaryFile(final String binaryFilename)
throws FileNotFoundException, IOException, UnsupportedFormatException { throws FileNotFoundException, IOException, UnsupportedFormatException {
final File file = new File(binaryFilename); final File file = new File(binaryFilename);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file); final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, 0, file.length());
return dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */); return dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */);
} }

3
tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java
View File

@ -77,7 +77,8 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase {
assertEquals("Wrong decode spec", BinaryDictOffdeviceUtils.COMPRESSION, step); assertEquals("Wrong decode spec", BinaryDictOffdeviceUtils.COMPRESSION, step);
} }
assertEquals("Wrong decode spec", 3, decodeSpec.mDecoderSpec.size()); assertEquals("Wrong decode spec", 3, decodeSpec.mDecoderSpec.size());
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(decodeSpec.mFile); final DictDecoder dictDecoder = FormatSpec.getDictDecoder(decodeSpec.mFile, 0,
decodeSpec.mFile.length());
final FusionDictionary resultDict = final FusionDictionary resultDict =
dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */); dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */);
assertEquals("Wrong version attribute", VERSION, resultDict.mOptions.mAttributes.get( assertEquals("Wrong version attribute", VERSION, resultDict.mOptions.mAttributes.get(