Merge "Fix auto-detection of format 4."
This commit is contained in:
commit
d7337a72bc
9 changed files with 75 additions and 62 deletions
|
@ -32,6 +32,10 @@ import java.util.TreeMap;
|
|||
* A base class of the binary dictionary decoder.
|
||||
*/
|
||||
public abstract class AbstractDictDecoder implements DictDecoder {
|
||||
private static final int SUCCESS = 0;
|
||||
private static final int ERROR_CANNOT_READ = 1;
|
||||
private static final int ERROR_WRONG_FORMAT = 2;
|
||||
|
||||
protected FileHeader readHeader(final DictBuffer dictBuffer)
|
||||
throws IOException, UnsupportedFormatException {
|
||||
if (dictBuffer == null) {
|
||||
|
@ -204,4 +208,25 @@ public abstract class AbstractDictDecoder implements DictDecoder {
|
|||
return readLength;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether the header contains the expected information. This is a no-error method,
|
||||
* that will return an error code and never throw a checked exception.
|
||||
* @return an error code, either ERROR_* or SUCCESS.
|
||||
*/
|
||||
private int checkHeader() {
|
||||
try {
|
||||
readHeader();
|
||||
} catch (IOException e) {
|
||||
return ERROR_CANNOT_READ;
|
||||
} catch (UnsupportedFormatException e) {
|
||||
return ERROR_WRONG_FORMAT;
|
||||
}
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasValidRawBinaryDictionary() {
|
||||
return checkHeader() == SUCCESS;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -24,12 +24,9 @@ import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
|
|||
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
|
@ -639,32 +636,10 @@ public final class BinaryDictDecoderUtils {
|
|||
/**
|
||||
* Basic test to find out whether the file is a binary dictionary or not.
|
||||
*
|
||||
* Concretely this only tests the magic number.
|
||||
*
|
||||
* @param file The file to test.
|
||||
* @return true if it's a binary dictionary, false otherwise
|
||||
*/
|
||||
public static boolean isBinaryDictionary(final File file) {
|
||||
FileInputStream inStream = null;
|
||||
try {
|
||||
inStream = new FileInputStream(file);
|
||||
final ByteBuffer buffer = inStream.getChannel().map(
|
||||
FileChannel.MapMode.READ_ONLY, 0, file.length());
|
||||
final int version = getFormatVersion(new ByteBufferDictBuffer(buffer));
|
||||
return (version >= FormatSpec.MINIMUM_SUPPORTED_VERSION
|
||||
&& version <= FormatSpec.MAXIMUM_SUPPORTED_VERSION);
|
||||
} catch (FileNotFoundException e) {
|
||||
return false;
|
||||
} catch (IOException e) {
|
||||
return false;
|
||||
} finally {
|
||||
if (inStream != null) {
|
||||
try {
|
||||
inStream.close();
|
||||
} catch (IOException e) {
|
||||
// do nothing
|
||||
}
|
||||
}
|
||||
}
|
||||
return FormatSpec.getDictDecoder(file).hasValidRawBinaryDictionary();
|
||||
}
|
||||
}
|
||||
|
|
|
@ -128,7 +128,8 @@ public interface DictDecoder {
|
|||
* Opens the dictionary file and makes DictBuffer.
|
||||
*/
|
||||
@UsedForTesting
|
||||
public void openDictBuffer() throws FileNotFoundException, IOException;
|
||||
public void openDictBuffer() throws FileNotFoundException, IOException,
|
||||
UnsupportedFormatException;
|
||||
@UsedForTesting
|
||||
public boolean isDictBufferOpen();
|
||||
|
||||
|
@ -229,4 +230,9 @@ public interface DictDecoder {
|
|||
}
|
||||
|
||||
public void skipPtNode(final FormatOptions formatOptions);
|
||||
|
||||
/**
|
||||
* @return whether this decoder has a valid binary dictionary that it can decode.
|
||||
*/
|
||||
public boolean hasValidRawBinaryDictionary();
|
||||
}
|
||||
|
|
|
@ -103,7 +103,7 @@ public class Ver4DictDecoder extends AbstractDictDecoder {
|
|||
mDictBuffer = mFrequencyBuffer = null;
|
||||
}
|
||||
|
||||
protected File getFile(final int fileType) {
|
||||
protected File getFile(final int fileType) throws UnsupportedFormatException {
|
||||
if (fileType == FILETYPE_TRIE) {
|
||||
return new File(mDictDirectory,
|
||||
mDictDirectory.getName() + FormatSpec.TRIE_FILE_EXTENSION);
|
||||
|
@ -122,12 +122,16 @@ public class Ver4DictDecoder extends AbstractDictDecoder {
|
|||
mDictDirectory.getName() + FormatSpec.SHORTCUT_FILE_EXTENSION
|
||||
+ FormatSpec.SHORTCUT_CONTENT_ID);
|
||||
} else {
|
||||
throw new RuntimeException("Unsupported kind of file : " + fileType);
|
||||
throw new UnsupportedFormatException("Unsupported kind of file : " + fileType);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void openDictBuffer() throws FileNotFoundException, IOException {
|
||||
public void openDictBuffer() throws FileNotFoundException, IOException,
|
||||
UnsupportedFormatException {
|
||||
if (!mDictDirectory.isDirectory()) {
|
||||
throw new UnsupportedFormatException("Format 4 dictionary needs a directory");
|
||||
}
|
||||
mDictBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_TRIE));
|
||||
mFrequencyBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_FREQUENCY));
|
||||
mTerminalAddressTableBuffer = mBufferFactory.getDictionaryBuffer(
|
||||
|
|
|
@ -45,7 +45,8 @@ public class Ver4DictUpdater extends Ver4DictDecoder implements DictUpdater {
|
|||
private final File mFrequencyFile;
|
||||
|
||||
@UsedForTesting
|
||||
public Ver4DictUpdater(final File dictDirectory, final int factoryType) {
|
||||
public Ver4DictUpdater(final File dictDirectory, final int factoryType)
|
||||
throws UnsupportedFormatException {
|
||||
// DictUpdater must have an updatable DictBuffer.
|
||||
super(dictDirectory, ((factoryType & MASK_DICTBUFFER) == USE_BYTEARRAY)
|
||||
? USE_BYTEARRAY : USE_WRITABLE_BYTEBUFFER);
|
||||
|
@ -664,7 +665,8 @@ public class Ver4DictUpdater extends Ver4DictDecoder implements DictUpdater {
|
|||
frequencyStream.close();
|
||||
}
|
||||
|
||||
private void insertTerminalPosition(final int posOfTerminal) throws IOException {
|
||||
private void insertTerminalPosition(final int posOfTerminal) throws IOException,
|
||||
UnsupportedFormatException {
|
||||
final OutputStream terminalPosStream = new FileOutputStream(
|
||||
getFile(FILETYPE_TERMINAL_ADDRESS_TABLE), true /* append */);
|
||||
BinaryDictEncoderUtils.writeUIntToStream(terminalPosStream, posOfTerminal,
|
||||
|
@ -702,7 +704,7 @@ public class Ver4DictUpdater extends Ver4DictDecoder implements DictUpdater {
|
|||
updater.insertShortcuts(terminalId, shortcuts);
|
||||
}
|
||||
|
||||
private void openBuffersAndStream() throws IOException {
|
||||
private void openBuffersAndStream() throws IOException, UnsupportedFormatException {
|
||||
openDictBuffer();
|
||||
mDictStream = new FileOutputStream(getFile(FILETYPE_TRIE), true /* append */);
|
||||
}
|
||||
|
|
|
@ -28,6 +28,7 @@ import com.android.inputmethod.latin.ExpandableBinaryDictionary;
|
|||
import com.android.inputmethod.latin.LatinImeLogger;
|
||||
import com.android.inputmethod.latin.makedict.DictDecoder;
|
||||
import com.android.inputmethod.latin.makedict.FormatSpec;
|
||||
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
|
||||
import com.android.inputmethod.latin.settings.Settings;
|
||||
import com.android.inputmethod.latin.utils.CollectionUtils;
|
||||
import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils;
|
||||
|
@ -222,6 +223,8 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
|
|||
UserHistoryDictIOUtils.readDictionaryBinary(dictDecoder, listener);
|
||||
} catch (IOException e) {
|
||||
Log.d(TAG, "IOException on opening a bytebuffer", e);
|
||||
} catch (UnsupportedFormatException e) {
|
||||
Log.d(TAG, "Unsupported format, can't read the dictionary", e);
|
||||
} finally {
|
||||
if (PROFILE_SAVE_RESTORE) {
|
||||
final long diff = System.currentTimeMillis() - now;
|
||||
|
@ -291,6 +294,8 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
|
|||
UserHistoryDictIOUtils.readDictionaryBinary(dictDecoder, listener);
|
||||
} catch (IOException e) {
|
||||
Log.d(TAG, "IOException on opening a bytebuffer", e);
|
||||
} catch (UnsupportedFormatException e) {
|
||||
Log.d(TAG, "Unsupported format, can't read the dictionary", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -29,6 +29,7 @@ import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
|
|||
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
|
||||
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
|
||||
import com.android.inputmethod.latin.utils.ByteArrayDictBuffer;
|
||||
import com.android.inputmethod.latin.utils.CollectionUtils;
|
||||
|
||||
|
@ -563,7 +564,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
|
|||
try {
|
||||
dictDecoder.openDictBuffer();
|
||||
} catch (IOException e) {
|
||||
// ignore
|
||||
Log.e(TAG, "IOException while opening the buffer", e);
|
||||
} catch (UnsupportedFormatException e) {
|
||||
Log.e(TAG, "IOException while opening the buffer", e);
|
||||
}
|
||||
assertTrue("Can't get the buffer", dictDecoder.isDictBufferOpen());
|
||||
|
@ -639,7 +641,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
|
|||
}
|
||||
}
|
||||
|
||||
private void runTestDeleteWord(final FormatOptions formatOptions) {
|
||||
private void runTestDeleteWord(final FormatOptions formatOptions)
|
||||
throws IOException, UnsupportedFormatException {
|
||||
final String dictName = "testDeleteWord";
|
||||
final String dictVersion = Long.toString(System.currentTimeMillis());
|
||||
final File file = BinaryDictUtils.getDictFile(dictName, dictVersion, formatOptions,
|
||||
|
@ -652,8 +655,6 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
|
|||
timeWritingDictToFile(file, dict, formatOptions);
|
||||
|
||||
final DictUpdater dictUpdater = BinaryDictUtils.getDictUpdater(file, formatOptions);
|
||||
|
||||
try {
|
||||
MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD,
|
||||
dictUpdater.getTerminalPosition(sWords.get(0)));
|
||||
dictUpdater.deleteWord(sWords.get(0));
|
||||
|
@ -665,12 +666,9 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
|
|||
dictUpdater.deleteWord(sWords.get(5));
|
||||
assertEquals(FormatSpec.NOT_VALID_WORD,
|
||||
dictUpdater.getTerminalPosition(sWords.get(5)));
|
||||
} catch (IOException e) {
|
||||
} catch (UnsupportedFormatException e) {
|
||||
}
|
||||
}
|
||||
|
||||
public void testDeleteWord() {
|
||||
public void testDeleteWord() throws IOException, UnsupportedFormatException {
|
||||
runTestDeleteWord(BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE);
|
||||
runTestDeleteWord(BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE);
|
||||
}
|
||||
|
|
|
@ -73,13 +73,14 @@ public class BinaryDictUtils {
|
|||
}
|
||||
}
|
||||
|
||||
public static DictUpdater getDictUpdater(final File file, final FormatOptions formatOptions) {
|
||||
public static DictUpdater getDictUpdater(final File file, final FormatOptions formatOptions)
|
||||
throws UnsupportedFormatException {
|
||||
if (formatOptions.mVersion == FormatSpec.VERSION4) {
|
||||
return new Ver4DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER);
|
||||
} else if (formatOptions.mVersion == 3) {
|
||||
return new Ver3DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER);
|
||||
} else {
|
||||
throw new RuntimeException("The format option has a wrong version : "
|
||||
throw new UnsupportedFormatException("The format option has a wrong version : "
|
||||
+ formatOptions.mVersion);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -26,6 +26,7 @@ import com.android.inputmethod.latin.makedict.DictEncoder;
|
|||
import com.android.inputmethod.latin.makedict.FormatSpec;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
|
||||
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
|
||||
import com.android.inputmethod.latin.makedict.Ver3DictDecoder;
|
||||
import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
|
||||
import com.android.inputmethod.latin.personalization.UserHistoryDictionaryBigramList;
|
||||
|
@ -142,15 +143,10 @@ public class UserHistoryDictIOUtilsTests extends AndroidTestCase
|
|||
UserHistoryDictIOUtils.writeDictionary(dictEncoder, this, bigramList, FORMAT_OPTIONS);
|
||||
}
|
||||
|
||||
private void readDictFromFile(final File file, final OnAddWordListener listener) {
|
||||
private void readDictFromFile(final File file, final OnAddWordListener listener)
|
||||
throws IOException, FileNotFoundException, UnsupportedFormatException {
|
||||
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, DictDecoder.USE_BYTEARRAY);
|
||||
try {
|
||||
dictDecoder.openDictBuffer();
|
||||
} catch (FileNotFoundException e) {
|
||||
Log.e(TAG, "file not found", e);
|
||||
} catch (IOException e) {
|
||||
Log.e(TAG, "IOException", e);
|
||||
}
|
||||
UserHistoryDictIOUtils.readDictionaryBinary(dictDecoder, listener);
|
||||
}
|
||||
|
||||
|
@ -169,7 +165,8 @@ public class UserHistoryDictIOUtilsTests extends AndroidTestCase
|
|||
checkWordsInFusionDict(fusionDict, addedWords);
|
||||
}
|
||||
|
||||
public void testReadAndWrite() {
|
||||
public void testReadAndWrite() throws IOException, FileNotFoundException,
|
||||
UnsupportedFormatException {
|
||||
final Context context = getContext();
|
||||
|
||||
File file = null;
|
||||
|
|
Loading…
Reference in a new issue