nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge "Fix auto-detection of format 4."

main
Jean Chalard 2013-11-29 10:31:36 +00:00 committed by Android (Google) Code Review
parent 3f8c1d4a19 26c9af33a7
commit d7337a72bc
9 changed files with 75 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java
View File

@ -32,6 +32,10 @@ import java.util.TreeMap;
* A base class of the binary dictionary decoder.
*/
public abstract class AbstractDictDecoder implements DictDecoder {
private static final int SUCCESS = 0;
private static final int ERROR_CANNOT_READ = 1;
private static final int ERROR_WRONG_FORMAT = 2;
protected FileHeader readHeader(final DictBuffer dictBuffer)
throws IOException, UnsupportedFormatException {
if (dictBuffer == null) {
@ -204,4 +208,25 @@ public abstract class AbstractDictDecoder implements DictDecoder {
return readLength;
}
}
/**
* Check whether the header contains the expected information. This is a no-error method,
* that will return an error code and never throw a checked exception.
* @return an error code, either ERROR_* or SUCCESS.
*/
private int checkHeader() {
try {
readHeader();
} catch (IOException e) {
return ERROR_CANNOT_READ;
} catch (UnsupportedFormatException e) {
return ERROR_WRONG_FORMAT;
}
return SUCCESS;
}
@Override
public boolean hasValidRawBinaryDictionary() {
return checkHeader() == SUCCESS;
}
}

27
java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java
View File

@ -24,12 +24,9 @@ import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.Map;
import java.util.TreeMap;
@ -639,32 +636,10 @@ public final class BinaryDictDecoderUtils {
/**
* Basic test to find out whether the file is a binary dictionary or not.
*
* Concretely this only tests the magic number.
*
* @param file The file to test.
* @return true if it's a binary dictionary, false otherwise
*/
public static boolean isBinaryDictionary(final File file) {
FileInputStream inStream = null;
try {
inStream = new FileInputStream(file);
final ByteBuffer buffer = inStream.getChannel().map(
FileChannel.MapMode.READ_ONLY, 0, file.length());
final int version = getFormatVersion(new ByteBufferDictBuffer(buffer));
return (version >= FormatSpec.MINIMUM_SUPPORTED_VERSION
&& version <= FormatSpec.MAXIMUM_SUPPORTED_VERSION);
} catch (FileNotFoundException e) {
return false;
} catch (IOException e) {
return false;
} finally {
if (inStream != null) {
try {
inStream.close();
} catch (IOException e) {
// do nothing
}
}
}
return FormatSpec.getDictDecoder(file).hasValidRawBinaryDictionary();
}
}

8
java/src/com/android/inputmethod/latin/makedict/DictDecoder.java
View File

@ -128,7 +128,8 @@ public interface DictDecoder {
* Opens the dictionary file and makes DictBuffer.
*/
@UsedForTesting
public void openDictBuffer() throws FileNotFoundException, IOException;
public void openDictBuffer() throws FileNotFoundException, IOException,
UnsupportedFormatException;
@UsedForTesting
public boolean isDictBufferOpen();
@ -229,4 +230,9 @@ public interface DictDecoder {
}
public void skipPtNode(final FormatOptions formatOptions);
/**
* @return whether this decoder has a valid binary dictionary that it can decode.
*/
public boolean hasValidRawBinaryDictionary();
}

10
java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java
View File

@ -103,7 +103,7 @@ public class Ver4DictDecoder extends AbstractDictDecoder {
mDictBuffer = mFrequencyBuffer = null;
}
protected File getFile(final int fileType) {
protected File getFile(final int fileType) throws UnsupportedFormatException {
if (fileType == FILETYPE_TRIE) {
return new File(mDictDirectory,
mDictDirectory.getName() + FormatSpec.TRIE_FILE_EXTENSION);
@ -122,12 +122,16 @@ public class Ver4DictDecoder extends AbstractDictDecoder {
mDictDirectory.getName() + FormatSpec.SHORTCUT_FILE_EXTENSION
+ FormatSpec.SHORTCUT_CONTENT_ID);
} else {
throw new RuntimeException("Unsupported kind of file : " + fileType);
throw new UnsupportedFormatException("Unsupported kind of file : " + fileType);
}
}
@Override
public void openDictBuffer() throws FileNotFoundException, IOException {
public void openDictBuffer() throws FileNotFoundException, IOException,
UnsupportedFormatException {
if (!mDictDirectory.isDirectory()) {
throw new UnsupportedFormatException("Format 4 dictionary needs a directory");
}
mDictBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_TRIE));
mFrequencyBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_FREQUENCY));
mTerminalAddressTableBuffer = mBufferFactory.getDictionaryBuffer(

8
java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java
View File

@ -45,7 +45,8 @@ public class Ver4DictUpdater extends Ver4DictDecoder implements DictUpdater {
private final File mFrequencyFile;
@UsedForTesting
public Ver4DictUpdater(final File dictDirectory, final int factoryType) {
public Ver4DictUpdater(final File dictDirectory, final int factoryType)
throws UnsupportedFormatException {
// DictUpdater must have an updatable DictBuffer.
super(dictDirectory, ((factoryType & MASK_DICTBUFFER) == USE_BYTEARRAY)
? USE_BYTEARRAY : USE_WRITABLE_BYTEBUFFER);
@ -664,7 +665,8 @@ public class Ver4DictUpdater extends Ver4DictDecoder implements DictUpdater {
frequencyStream.close();
}
private void insertTerminalPosition(final int posOfTerminal) throws IOException {
private void insertTerminalPosition(final int posOfTerminal) throws IOException,
UnsupportedFormatException {
final OutputStream terminalPosStream = new FileOutputStream(
getFile(FILETYPE_TERMINAL_ADDRESS_TABLE), true /* append */);
BinaryDictEncoderUtils.writeUIntToStream(terminalPosStream, posOfTerminal,
@ -702,7 +704,7 @@ public class Ver4DictUpdater extends Ver4DictDecoder implements DictUpdater {
updater.insertShortcuts(terminalId, shortcuts);
}
private void openBuffersAndStream() throws IOException {
private void openBuffersAndStream() throws IOException, UnsupportedFormatException {
openDictBuffer();
mDictStream = new FileOutputStream(getFile(FILETYPE_TRIE), true /* append */);
}

5
java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
View File

@ -28,6 +28,7 @@ import com.android.inputmethod.latin.ExpandableBinaryDictionary;
import com.android.inputmethod.latin.LatinImeLogger;
import com.android.inputmethod.latin.makedict.DictDecoder;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.settings.Settings;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils;
@ -222,6 +223,8 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
UserHistoryDictIOUtils.readDictionaryBinary(dictDecoder, listener);
} catch (IOException e) {
Log.d(TAG, "IOException on opening a bytebuffer", e);
} catch (UnsupportedFormatException e) {
Log.d(TAG, "Unsupported format, can't read the dictionary", e);
} finally {
if (PROFILE_SAVE_RESTORE) {
final long diff = System.currentTimeMillis() - now;
@ -291,6 +294,8 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
UserHistoryDictIOUtils.readDictionaryBinary(dictDecoder, listener);
} catch (IOException e) {
Log.d(TAG, "IOException on opening a bytebuffer", e);
} catch (UnsupportedFormatException e) {
Log.d(TAG, "Unsupported format, can't read the dictionary", e);
}
}

34
tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java
View File

@ -29,6 +29,7 @@ import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.utils.ByteArrayDictBuffer;
import com.android.inputmethod.latin.utils.CollectionUtils;
@ -563,7 +564,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
try {
dictDecoder.openDictBuffer();
} catch (IOException e) {
// ignore
Log.e(TAG, "IOException while opening the buffer", e);
} catch (UnsupportedFormatException e) {
Log.e(TAG, "IOException while opening the buffer", e);
}
assertTrue("Can't get the buffer", dictDecoder.isDictBufferOpen());
@ -639,7 +641,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
}
}
private void runTestDeleteWord(final FormatOptions formatOptions) {
private void runTestDeleteWord(final FormatOptions formatOptions)
throws IOException, UnsupportedFormatException {
final String dictName = "testDeleteWord";
final String dictVersion = Long.toString(System.currentTimeMillis());
final File file = BinaryDictUtils.getDictFile(dictName, dictVersion, formatOptions,
@ -652,25 +655,20 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
timeWritingDictToFile(file, dict, formatOptions);
final DictUpdater dictUpdater = BinaryDictUtils.getDictUpdater(file, formatOptions);
MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD,
dictUpdater.getTerminalPosition(sWords.get(0)));
dictUpdater.deleteWord(sWords.get(0));
assertEquals(FormatSpec.NOT_VALID_WORD,
dictUpdater.getTerminalPosition(sWords.get(0)));
try {
MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD,
dictUpdater.getTerminalPosition(sWords.get(0)));
dictUpdater.deleteWord(sWords.get(0));
assertEquals(FormatSpec.NOT_VALID_WORD,
dictUpdater.getTerminalPosition(sWords.get(0)));
MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD,
dictUpdater.getTerminalPosition(sWords.get(5)));
dictUpdater.deleteWord(sWords.get(5));
assertEquals(FormatSpec.NOT_VALID_WORD,
dictUpdater.getTerminalPosition(sWords.get(5)));
} catch (IOException e) {
} catch (UnsupportedFormatException e) {
}
MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD,
dictUpdater.getTerminalPosition(sWords.get(5)));
dictUpdater.deleteWord(sWords.get(5));
assertEquals(FormatSpec.NOT_VALID_WORD,
dictUpdater.getTerminalPosition(sWords.get(5)));
}
public void testDeleteWord() {
public void testDeleteWord() throws IOException, UnsupportedFormatException {
runTestDeleteWord(BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE);
runTestDeleteWord(BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE);
}

5
tests/src/com/android/inputmethod/latin/makedict/BinaryDictUtils.java
View File

@ -73,13 +73,14 @@ public class BinaryDictUtils {
}
}
public static DictUpdater getDictUpdater(final File file, final FormatOptions formatOptions) {
public static DictUpdater getDictUpdater(final File file, final FormatOptions formatOptions)
throws UnsupportedFormatException {
if (formatOptions.mVersion == FormatSpec.VERSION4) {
return new Ver4DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER);
} else if (formatOptions.mVersion == 3) {
return new Ver3DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER);
} else {
throw new RuntimeException("The format option has a wrong version : "
throw new UnsupportedFormatException("The format option has a wrong version : "
+ formatOptions.mVersion);
}
}

15
tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java
View File

@ -26,6 +26,7 @@ import com.android.inputmethod.latin.makedict.DictEncoder;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.makedict.Ver3DictDecoder;
import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
import com.android.inputmethod.latin.personalization.UserHistoryDictionaryBigramList;
@ -142,15 +143,10 @@ public class UserHistoryDictIOUtilsTests extends AndroidTestCase
UserHistoryDictIOUtils.writeDictionary(dictEncoder, this, bigramList, FORMAT_OPTIONS);
}
private void readDictFromFile(final File file, final OnAddWordListener listener) {
private void readDictFromFile(final File file, final OnAddWordListener listener)
throws IOException, FileNotFoundException, UnsupportedFormatException {
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, DictDecoder.USE_BYTEARRAY);
try {
dictDecoder.openDictBuffer();
} catch (FileNotFoundException e) {
Log.e(TAG, "file not found", e);
} catch (IOException e) {
Log.e(TAG, "IOException", e);
}
dictDecoder.openDictBuffer();
UserHistoryDictIOUtils.readDictionaryBinary(dictDecoder, listener);
}
@ -169,7 +165,8 @@ public class UserHistoryDictIOUtilsTests extends AndroidTestCase
checkWordsInFusionDict(fusionDict, addedWords);
}
public void testReadAndWrite() {
public void testReadAndWrite() throws IOException, FileNotFoundException,
UnsupportedFormatException {
final Context context = getContext();
File file = null;