am a6dc3056: Merge "Add getDictDecoder."
* commit 'a6dc3056ffefd40e8fa93060d096c80889e82633': Add getDictDecoder.main
commit
712fb6c613
|
@ -21,9 +21,10 @@ import android.content.SharedPreferences;
|
|||
import android.content.res.AssetFileDescriptor;
|
||||
import android.util.Log;
|
||||
|
||||
import com.android.inputmethod.latin.makedict.DictDecoder;
|
||||
import com.android.inputmethod.latin.makedict.FormatSpec;
|
||||
import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
|
||||
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
|
||||
import com.android.inputmethod.latin.makedict.Ver3DictDecoder;
|
||||
import com.android.inputmethod.latin.utils.CollectionUtils;
|
||||
import com.android.inputmethod.latin.utils.DictionaryInfoUtils;
|
||||
import com.android.inputmethod.latin.utils.LocaleUtils;
|
||||
|
@ -228,7 +229,7 @@ final public class BinaryDictionaryGetter {
|
|||
private static boolean hackCanUseDictionaryFile(final Locale locale, final File f) {
|
||||
try {
|
||||
// Read the version of the file
|
||||
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(f);
|
||||
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(f);
|
||||
final FileHeader header = dictDecoder.readHeader();
|
||||
|
||||
final String version = header.mDictionaryOptions.mAttributes.get(VERSION_KEY);
|
||||
|
|
|
@ -148,7 +148,7 @@ public final class BinaryDictIOUtils {
|
|||
* @throws IOException if the file can't be read.
|
||||
* @throws UnsupportedFormatException if the format of the file is not recognized.
|
||||
*/
|
||||
/* package */ static void readUnigramsAndBigramsBinary(final Ver3DictDecoder dictDecoder,
|
||||
/* package */ static void readUnigramsAndBigramsBinary(final DictDecoder dictDecoder,
|
||||
final Map<Integer, String> words, final Map<Integer, Integer> frequencies,
|
||||
final Map<Integer, ArrayList<PendingAttribute>> bigrams) throws IOException,
|
||||
UnsupportedFormatException {
|
||||
|
@ -169,7 +169,7 @@ public final class BinaryDictIOUtils {
|
|||
* @throws UnsupportedFormatException if the format of the file is not recognized.
|
||||
*/
|
||||
@UsedForTesting
|
||||
/* package */ static int getTerminalPosition(final Ver3DictDecoder dictDecoder,
|
||||
/* package */ static int getTerminalPosition(final DictDecoder dictDecoder,
|
||||
final String word) throws IOException, UnsupportedFormatException {
|
||||
if (word == null) return FormatSpec.NOT_VALID_WORD;
|
||||
dictDecoder.setPosition(0);
|
||||
|
@ -519,7 +519,7 @@ public final class BinaryDictIOUtils {
|
|||
final File file, final long offset, final long length)
|
||||
throws FileNotFoundException, IOException, UnsupportedFormatException {
|
||||
final byte[] buffer = new byte[HEADER_READING_BUFFER_SIZE];
|
||||
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file,
|
||||
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file,
|
||||
new DictDecoder.DictionaryBufferFactory() {
|
||||
@Override
|
||||
public DictBuffer getDictionaryBuffer(File file)
|
||||
|
|
|
@ -118,6 +118,14 @@ public interface DictDecoder {
|
|||
public boolean readForwardLinkAndAdvancePosition();
|
||||
public boolean hasNextPtNodeArray();
|
||||
|
||||
/**
|
||||
* Opens the dictionary file and makes DictBuffer.
|
||||
*/
|
||||
@UsedForTesting
|
||||
public void openDictBuffer() throws FileNotFoundException, IOException;
|
||||
@UsedForTesting
|
||||
public boolean isOpenedDictBuffer();
|
||||
|
||||
// Flags for DictionaryBufferFactory.
|
||||
public static final int USE_READONLY_BYTEBUFFER = 0x01000000;
|
||||
public static final int USE_BYTEARRAY = 0x02000000;
|
||||
|
|
|
@ -18,8 +18,11 @@ package com.android.inputmethod.latin.makedict;
|
|||
|
||||
import com.android.inputmethod.annotations.UsedForTesting;
|
||||
import com.android.inputmethod.latin.Constants;
|
||||
import com.android.inputmethod.latin.makedict.DictDecoder.DictionaryBufferFactory;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
/**
|
||||
* Dictionary File Format Specification.
|
||||
*/
|
||||
|
@ -341,6 +344,28 @@ public final class FormatSpec {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns new dictionary decoder.
|
||||
*
|
||||
* @param dictFile the dictionary file.
|
||||
* @param bufferType the flag indicating buffer type which is used by the dictionary decoder.
|
||||
* @return new dictionary decoder if the dictionary file exists, otherwise null.
|
||||
*/
|
||||
public static DictDecoder getDictDecoder(final File dictFile, final int bufferType) {
|
||||
if (!dictFile.isFile()) return null;
|
||||
return new Ver3DictDecoder(dictFile, bufferType);
|
||||
}
|
||||
|
||||
public static DictDecoder getDictDecoder(final File dictFile,
|
||||
final DictionaryBufferFactory factory) {
|
||||
if (!dictFile.isFile()) return null;
|
||||
return new Ver3DictDecoder(dictFile, factory);
|
||||
}
|
||||
|
||||
public static DictDecoder getDictDecoder(final File dictFile) {
|
||||
return getDictDecoder(dictFile, DictDecoder.USE_READONLY_BYTEBUFFER);
|
||||
}
|
||||
|
||||
private FormatSpec() {
|
||||
// This utility class is not publicly instantiable.
|
||||
}
|
||||
|
|
|
@ -173,11 +173,7 @@ public class Ver3DictDecoder implements DictDecoder {
|
|||
private final DictionaryBufferFactory mBufferFactory;
|
||||
private DictBuffer mDictBuffer;
|
||||
|
||||
public Ver3DictDecoder(final File file) {
|
||||
this(file, USE_READONLY_BYTEBUFFER);
|
||||
}
|
||||
|
||||
public Ver3DictDecoder(final File file, final int factoryFlag) {
|
||||
/* package */ Ver3DictDecoder(final File file, final int factoryFlag) {
|
||||
mDictionaryBinaryFile = file;
|
||||
mDictBuffer = null;
|
||||
|
||||
|
@ -192,15 +188,21 @@ public class Ver3DictDecoder implements DictDecoder {
|
|||
}
|
||||
}
|
||||
|
||||
public Ver3DictDecoder(final File file, final DictionaryBufferFactory factory) {
|
||||
/* package */ Ver3DictDecoder(final File file, final DictionaryBufferFactory factory) {
|
||||
mDictionaryBinaryFile = file;
|
||||
mBufferFactory = factory;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void openDictBuffer() throws FileNotFoundException, IOException {
|
||||
mDictBuffer = mBufferFactory.getDictionaryBuffer(mDictionaryBinaryFile);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isOpenedDictBuffer() {
|
||||
return mDictBuffer != null;
|
||||
}
|
||||
|
||||
/* package */ DictBuffer getDictBuffer() {
|
||||
return mDictBuffer;
|
||||
}
|
||||
|
|
|
@ -25,14 +25,13 @@ import com.android.inputmethod.latin.Constants;
|
|||
import com.android.inputmethod.latin.ExpandableBinaryDictionary;
|
||||
import com.android.inputmethod.latin.LatinImeLogger;
|
||||
import com.android.inputmethod.latin.makedict.DictDecoder;
|
||||
import com.android.inputmethod.latin.makedict.Ver3DictDecoder;
|
||||
import com.android.inputmethod.latin.makedict.FormatSpec;
|
||||
import com.android.inputmethod.latin.settings.Settings;
|
||||
import com.android.inputmethod.latin.utils.CollectionUtils;
|
||||
import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils;
|
||||
import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListener;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
|
||||
|
@ -173,16 +172,19 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi
|
|||
|
||||
// Load the dictionary from binary file
|
||||
final File dictFile = new File(mContext.getFilesDir(), mFileName);
|
||||
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(dictFile,
|
||||
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(dictFile,
|
||||
DictDecoder.USE_BYTEARRAY);
|
||||
if (dictDecoder == null) {
|
||||
// This is an expected condition: we don't have a user history dictionary for this
|
||||
// language yet. It will be created sometime later.
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
dictDecoder.openDictBuffer();
|
||||
UserHistoryDictIOUtils.readDictionaryBinary(dictDecoder, listener);
|
||||
} catch (FileNotFoundException e) {
|
||||
// This is an expected condition: we don't have a user history dictionary for this
|
||||
// language yet. It will be created sometime later.
|
||||
} catch (IOException e) {
|
||||
Log.e(TAG, "IOException on opening a bytebuffer", e);
|
||||
Log.d(TAG, "IOException on opening a bytebuffer", e);
|
||||
} finally {
|
||||
if (PROFILE_SAVE_RESTORE) {
|
||||
final long diff = System.currentTimeMillis() - now;
|
||||
|
|
|
@ -20,13 +20,13 @@ import android.util.Log;
|
|||
|
||||
import com.android.inputmethod.annotations.UsedForTesting;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictIOUtils;
|
||||
import com.android.inputmethod.latin.makedict.DictDecoder;
|
||||
import com.android.inputmethod.latin.makedict.DictEncoder;
|
||||
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
|
||||
import com.android.inputmethod.latin.makedict.PendingAttribute;
|
||||
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
|
||||
import com.android.inputmethod.latin.makedict.Ver3DictDecoder;
|
||||
import com.android.inputmethod.latin.personalization.UserHistoryDictionaryBigramList;
|
||||
|
||||
import java.io.IOException;
|
||||
|
@ -125,7 +125,7 @@ public final class UserHistoryDictIOUtils {
|
|||
/**
|
||||
* Reads dictionary from file.
|
||||
*/
|
||||
public static void readDictionaryBinary(final Ver3DictDecoder dictDecoder,
|
||||
public static void readDictionaryBinary(final DictDecoder dictDecoder,
|
||||
final OnAddWordListener dict) {
|
||||
final TreeMap<Integer, String> unigrams = CollectionUtils.newTreeMap();
|
||||
final TreeMap<Integer, Integer> frequencies = CollectionUtils.newTreeMap();
|
||||
|
|
|
@ -131,18 +131,6 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
|
|||
|
||||
// Utilities for test
|
||||
|
||||
/**
|
||||
* Makes new DictDecoder according to BUFFER_TYPE.
|
||||
*/
|
||||
private Ver3DictDecoder getDictDecoder(final File file, final int bufferType) {
|
||||
if (bufferType == USE_BYTE_BUFFER) {
|
||||
return new Ver3DictDecoder(file, DictDecoder.USE_READONLY_BYTEBUFFER);
|
||||
} else if (bufferType == USE_BYTE_ARRAY) {
|
||||
return new Ver3DictDecoder(file, DictDecoder.USE_BYTEARRAY);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates a random word.
|
||||
*/
|
||||
|
@ -285,9 +273,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
|
|||
|
||||
FusionDictionary dict = null;
|
||||
try {
|
||||
final Ver3DictDecoder dictDecoder = getDictDecoder(file, bufferType);
|
||||
dictDecoder.openDictBuffer();
|
||||
assertNotNull(dictDecoder.getDictBuffer());
|
||||
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, bufferType);
|
||||
now = System.currentTimeMillis();
|
||||
dict = dictDecoder.readDictionaryBinary(null, false /* deleteDictIfBroken */);
|
||||
diff = System.currentTimeMillis() - now;
|
||||
|
@ -443,9 +429,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
|
|||
|
||||
long now = -1, diff = -1;
|
||||
try {
|
||||
final Ver3DictDecoder dictDecoder = getDictDecoder(file, bufferType);
|
||||
dictDecoder.openDictBuffer();
|
||||
assertNotNull("Can't get buffer.", dictDecoder.getDictBuffer());
|
||||
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, bufferType);
|
||||
now = System.currentTimeMillis();
|
||||
dictDecoder.readUnigramsAndBigramsBinary(resultWords, resultFreqs, resultBigrams);
|
||||
diff = System.currentTimeMillis() - now;
|
||||
|
@ -531,9 +515,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
|
|||
}
|
||||
|
||||
// Tests for getTerminalPosition
|
||||
private String getWordFromBinary(final Ver3DictDecoder dictDecoder, final int address) {
|
||||
final DictBuffer dictBuffer = dictDecoder.getDictBuffer();
|
||||
if (dictBuffer.position() != 0) dictBuffer.position(0);
|
||||
private String getWordFromBinary(final DictDecoder dictDecoder, final int address) {
|
||||
if (dictDecoder.getPosition() != 0) dictDecoder.setPosition(0);
|
||||
|
||||
FileHeader fileHeader = null;
|
||||
try {
|
||||
|
@ -548,7 +531,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
|
|||
address, fileHeader.mFormatOptions).mWord;
|
||||
}
|
||||
|
||||
private long runGetTerminalPosition(final Ver3DictDecoder dictDecoder, final String word,
|
||||
private long runGetTerminalPosition(final DictDecoder dictDecoder, final String word,
|
||||
int index, boolean contained) {
|
||||
final int expectedFrequency = (UNIGRAM_FREQ + index) % 255;
|
||||
long diff = -1;
|
||||
|
@ -584,14 +567,14 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
|
|||
addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */);
|
||||
timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE);
|
||||
|
||||
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, DictDecoder.USE_BYTEARRAY);
|
||||
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, DictDecoder.USE_BYTEARRAY);
|
||||
try {
|
||||
dictDecoder.openDictBuffer();
|
||||
} catch (IOException e) {
|
||||
// ignore
|
||||
Log.e(TAG, "IOException while opening the buffer", e);
|
||||
}
|
||||
assertNotNull("Can't get the buffer", dictDecoder.getDictBuffer());
|
||||
assertTrue("Can't get the buffer", dictDecoder.isOpenedDictBuffer());
|
||||
|
||||
try {
|
||||
// too long word
|
||||
|
@ -648,7 +631,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
|
|||
// ignore
|
||||
Log.e(TAG, "IOException while opening the buffer", e);
|
||||
}
|
||||
assertNotNull("Can't get the buffer", dictDecoder.getDictBuffer());
|
||||
assertTrue("Can't get the buffer", dictDecoder.isOpenedDictBuffer());
|
||||
|
||||
try {
|
||||
MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD,
|
||||
|
|
|
@ -140,7 +140,8 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
|
|||
int position = FormatSpec.NOT_VALID_WORD;
|
||||
|
||||
try {
|
||||
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
|
||||
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file,
|
||||
DictDecoder.USE_READONLY_BYTEBUFFER);
|
||||
position = dictDecoder.getTerminalPosition(word);
|
||||
} catch (IOException e) {
|
||||
} catch (UnsupportedFormatException e) {
|
||||
|
@ -149,7 +150,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
|
|||
}
|
||||
|
||||
/**
|
||||
* Find a word using the Ver3DictDecoder.
|
||||
* Find a word using the DictDecoder.
|
||||
*
|
||||
* @param dictDecoder the dict decoder
|
||||
* @param word the word searched
|
||||
|
@ -157,21 +158,20 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
|
|||
* @throws IOException
|
||||
* @throws UnsupportedFormatException
|
||||
*/
|
||||
private static PtNodeInfo findWordByBinaryDictReader(final Ver3DictDecoder dictDecoder,
|
||||
private static PtNodeInfo findWordByBinaryDictReader(final DictDecoder dictDecoder,
|
||||
final String word) throws IOException, UnsupportedFormatException {
|
||||
int position = dictDecoder.getTerminalPosition(word);
|
||||
final DictBuffer dictBuffer = dictDecoder.getDictBuffer();
|
||||
if (position != FormatSpec.NOT_VALID_WORD) {
|
||||
dictBuffer.position(0);
|
||||
dictDecoder.setPosition(0);
|
||||
final FileHeader header = dictDecoder.readHeader();
|
||||
dictBuffer.position(position);
|
||||
dictDecoder.setPosition(position);
|
||||
return dictDecoder.readPtNode(position, header.mFormatOptions);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private PtNodeInfo findWordFromFile(final File file, final String word) {
|
||||
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
|
||||
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file);
|
||||
PtNodeInfo info = null;
|
||||
try {
|
||||
dictDecoder.openDictBuffer();
|
||||
|
@ -234,7 +234,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
|
|||
private void checkReverseLookup(final File file, final String word, final int position) {
|
||||
|
||||
try {
|
||||
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
|
||||
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file);
|
||||
final FileHeader fileHeader = dictDecoder.readHeader();
|
||||
assertEquals(word,
|
||||
BinaryDictDecoderUtils.getWordAtPosition(dictDecoder, fileHeader.mHeaderSize,
|
||||
|
|
|
@ -143,7 +143,7 @@ public class UserHistoryDictIOUtilsTests extends AndroidTestCase
|
|||
}
|
||||
|
||||
private void readDictFromFile(final File file, final OnAddWordListener listener) {
|
||||
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, DictDecoder.USE_BYTEARRAY);
|
||||
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, DictDecoder.USE_BYTEARRAY);
|
||||
try {
|
||||
dictDecoder.openDictBuffer();
|
||||
} catch (FileNotFoundException e) {
|
||||
|
|
|
@ -18,9 +18,9 @@ package com.android.inputmethod.latin.dicttool;
|
|||
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils;
|
||||
import com.android.inputmethod.latin.makedict.DictDecoder;
|
||||
import com.android.inputmethod.latin.makedict.FormatSpec;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary;
|
||||
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
|
||||
import com.android.inputmethod.latin.makedict.Ver3DictDecoder;
|
||||
|
||||
import org.xml.sax.SAXException;
|
||||
|
||||
|
@ -185,7 +185,7 @@ public final class BinaryDictOffdeviceUtils {
|
|||
crash(filename, new RuntimeException(
|
||||
filename + " does not seem to be a dictionary file"));
|
||||
} else {
|
||||
final DictDecoder dictDecoder = new Ver3DictDecoder(decodedSpec.mFile,
|
||||
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file,
|
||||
DictDecoder.USE_BYTEARRAY);
|
||||
if (report) {
|
||||
System.out.println("Format : Binary dictionary format");
|
||||
|
|
|
@ -267,7 +267,7 @@ public class DictionaryMaker {
|
|||
private static FusionDictionary readBinaryFile(final String binaryFilename)
|
||||
throws FileNotFoundException, IOException, UnsupportedFormatException {
|
||||
final File file = new File(binaryFilename);
|
||||
final DictDecoder dictDecoder = new Ver3DictDecoder(file);
|
||||
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file);
|
||||
return dictDecoder.readDictionaryBinary(null, false /* deleteDictIfBroken */);
|
||||
}
|
||||
|
||||
|
|
|
@ -18,12 +18,12 @@ package com.android.inputmethod.latin.dicttool;
|
|||
|
||||
import com.android.inputmethod.latin.makedict.DictDecoder;
|
||||
import com.android.inputmethod.latin.makedict.DictEncoder;
|
||||
import com.android.inputmethod.latin.makedict.FormatSpec;
|
||||
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
|
||||
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
|
||||
import com.android.inputmethod.latin.makedict.Ver3DictDecoder;
|
||||
import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
|
||||
|
||||
import junit.framework.TestCase;
|
||||
|
@ -69,7 +69,7 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase {
|
|||
assertEquals("Wrong decode spec", BinaryDictOffdeviceUtils.COMPRESSION, step);
|
||||
}
|
||||
assertEquals("Wrong decode spec", 3, decodeSpec.mDecoderSpec.size());
|
||||
final DictDecoder dictDecoder = new Ver3DictDecoder(decodeSpec.mFile);
|
||||
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(decodeSpec.mFile);
|
||||
final FusionDictionary resultDict = dictDecoder.readDictionaryBinary(
|
||||
null /* dict : an optional dictionary to add words to, or null */,
|
||||
false /* deleteDictIfBroken */);
|
||||
|
|
Loading…
Reference in New Issue