Merge "Simplify dictionary creation in unit tests."
commit
73edc1eed9
|
@ -37,6 +37,7 @@ import java.io.File;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
|
import java.util.HashSet;
|
||||||
import java.util.Locale;
|
import java.util.Locale;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Random;
|
import java.util.Random;
|
||||||
|
@ -49,6 +50,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
|
||||||
private static final int DUMMY_PROBABILITY = 0;
|
private static final int DUMMY_PROBABILITY = 0;
|
||||||
private static final int[] DICT_FORMAT_VERSIONS =
|
private static final int[] DICT_FORMAT_VERSIONS =
|
||||||
new int[] { FormatSpec.VERSION4, FormatSpec.VERSION4_DEV };
|
new int[] { FormatSpec.VERSION4, FormatSpec.VERSION4_DEV };
|
||||||
|
private static final String DICTIONARY_ID = "TestDecayingBinaryDictionary";
|
||||||
|
|
||||||
private int mCurrentTime = 0;
|
private int mCurrentTime = 0;
|
||||||
|
|
||||||
|
@ -56,10 +58,15 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
|
||||||
protected void setUp() throws Exception {
|
protected void setUp() throws Exception {
|
||||||
super.setUp();
|
super.setUp();
|
||||||
mCurrentTime = 0;
|
mCurrentTime = 0;
|
||||||
|
mDictFilesToBeDeleted.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void tearDown() throws Exception {
|
protected void tearDown() throws Exception {
|
||||||
|
for (final File dictFile : mDictFilesToBeDeleted) {
|
||||||
|
dictFile.delete();
|
||||||
|
}
|
||||||
|
mDictFilesToBeDeleted.clear();
|
||||||
stopTestModeInNativeCode();
|
stopTestModeInNativeCode();
|
||||||
super.tearDown();
|
super.tearDown();
|
||||||
}
|
}
|
||||||
|
@ -103,25 +110,32 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
|
||||||
binaryDictionary.flushWithGC();
|
binaryDictionary.flushWithGC();
|
||||||
}
|
}
|
||||||
|
|
||||||
private File createEmptyDictionaryAndGetFile(final String dictId,
|
private HashSet<File> mDictFilesToBeDeleted = new HashSet<>();
|
||||||
final int formatVersion) throws IOException {
|
|
||||||
|
private File createEmptyDictionaryAndGetFile(final int formatVersion) {
|
||||||
if (formatVersion == FormatSpec.VERSION4
|
if (formatVersion == FormatSpec.VERSION4
|
||||||
|| formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING
|
|| formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING
|
||||||
|| formatVersion == FormatSpec.VERSION4_DEV) {
|
|| formatVersion == FormatSpec.VERSION4_DEV) {
|
||||||
return createEmptyVer4DictionaryAndGetFile(dictId, formatVersion);
|
try {
|
||||||
} else {
|
final File dictFile = createEmptyVer4DictionaryAndGetFile(formatVersion);
|
||||||
throw new IOException("Dictionary format version " + formatVersion
|
mDictFilesToBeDeleted.add(dictFile);
|
||||||
+ " is not supported.");
|
return dictFile;
|
||||||
|
} catch (final IOException e) {
|
||||||
|
fail(e.toString());
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
fail("Dictionary format version " + formatVersion + " is not supported.");
|
||||||
|
}
|
||||||
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
private File createEmptyVer4DictionaryAndGetFile(final String dictId, final int formatVersion)
|
private File createEmptyVer4DictionaryAndGetFile(final int formatVersion)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
final File file = File.createTempFile(dictId, TEST_DICT_FILE_EXTENSION,
|
final File file = File.createTempFile(DICTIONARY_ID, TEST_DICT_FILE_EXTENSION,
|
||||||
getContext().getCacheDir());
|
getContext().getCacheDir());
|
||||||
FileUtils.deleteRecursively(file);
|
FileUtils.deleteRecursively(file);
|
||||||
Map<String, String> attributeMap = new HashMap<>();
|
Map<String, String> attributeMap = new HashMap<>();
|
||||||
attributeMap.put(DictionaryHeader.DICTIONARY_ID_KEY, dictId);
|
attributeMap.put(DictionaryHeader.DICTIONARY_ID_KEY, DICTIONARY_ID);
|
||||||
attributeMap.put(DictionaryHeader.DICTIONARY_VERSION_KEY,
|
attributeMap.put(DictionaryHeader.DICTIONARY_VERSION_KEY,
|
||||||
String.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis())));
|
String.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis())));
|
||||||
attributeMap.put(DictionaryHeader.USES_FORGETTING_CURVE_KEY,
|
attributeMap.put(DictionaryHeader.USES_FORGETTING_CURVE_KEY,
|
||||||
|
@ -137,6 +151,12 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private BinaryDictionary getBinaryDictionary(final File dictFile) {
|
||||||
|
return new BinaryDictionary(dictFile.getAbsolutePath(),
|
||||||
|
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
||||||
|
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
||||||
|
}
|
||||||
|
|
||||||
private static int setCurrentTimeForTestMode(final int currentTime) {
|
private static int setCurrentTimeForTestMode(final int currentTime) {
|
||||||
return BinaryDictionaryUtils.setCurrentTimeForTest(currentTime);
|
return BinaryDictionaryUtils.setCurrentTimeForTest(currentTime);
|
||||||
}
|
}
|
||||||
|
@ -153,15 +173,8 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
|
||||||
|
|
||||||
private void testReadDictInJavaSide(final int formatVersion) {
|
private void testReadDictInJavaSide(final int formatVersion) {
|
||||||
setCurrentTimeForTestMode(mCurrentTime);
|
setCurrentTimeForTestMode(mCurrentTime);
|
||||||
File dictFile = null;
|
final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
|
||||||
try {
|
final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
|
addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
|
||||||
addUnigramWord(binaryDictionary, "ab", DUMMY_PROBABILITY);
|
addUnigramWord(binaryDictionary, "ab", DUMMY_PROBABILITY);
|
||||||
addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY);
|
addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY);
|
||||||
|
@ -189,7 +202,6 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
|
||||||
} catch (UnsupportedFormatException e) {
|
} catch (UnsupportedFormatException e) {
|
||||||
fail("Unsupported format: " + e);
|
fail("Unsupported format: " + e);
|
||||||
}
|
}
|
||||||
dictFile.delete();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testControlCurrentTime() {
|
public void testControlCurrentTime() {
|
||||||
|
@ -214,15 +226,8 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void testAddValidAndInvalidWords(final int formatVersion) {
|
private void testAddValidAndInvalidWords(final int formatVersion) {
|
||||||
File dictFile = null;
|
final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
|
||||||
try {
|
final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
|
|
||||||
addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY);
|
addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY);
|
||||||
assertFalse(binaryDictionary.isValidWord("a"));
|
assertFalse(binaryDictionary.isValidWord("a"));
|
||||||
|
@ -249,7 +254,6 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
|
||||||
assertFalse(isValidBigram(binaryDictionary, "x", "y"));
|
assertFalse(isValidBigram(binaryDictionary, "x", "y"));
|
||||||
|
|
||||||
binaryDictionary.close();
|
binaryDictionary.close();
|
||||||
dictFile.delete();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testDecayingProbability() {
|
public void testDecayingProbability() {
|
||||||
|
@ -259,15 +263,8 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void testDecayingProbability(final int formatVersion) {
|
private void testDecayingProbability(final int formatVersion) {
|
||||||
File dictFile = null;
|
final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
|
||||||
try {
|
final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
|
|
||||||
addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
|
addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
|
||||||
assertTrue(binaryDictionary.isValidWord("a"));
|
assertTrue(binaryDictionary.isValidWord("a"));
|
||||||
|
@ -306,7 +303,6 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
|
||||||
assertFalse(isValidBigram(binaryDictionary, "a", "b"));
|
assertFalse(isValidBigram(binaryDictionary, "a", "b"));
|
||||||
|
|
||||||
binaryDictionary.close();
|
binaryDictionary.close();
|
||||||
dictFile.delete();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testAddManyUnigramsToDecayingDict() {
|
public void testAddManyUnigramsToDecayingDict() {
|
||||||
|
@ -321,16 +317,8 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
|
||||||
final int codePointSetSize = 50;
|
final int codePointSetSize = 50;
|
||||||
final long seed = System.currentTimeMillis();
|
final long seed = System.currentTimeMillis();
|
||||||
final Random random = new Random(seed);
|
final Random random = new Random(seed);
|
||||||
|
final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
|
||||||
File dictFile = null;
|
final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
|
||||||
try {
|
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
setCurrentTimeForTestMode(mCurrentTime);
|
setCurrentTimeForTestMode(mCurrentTime);
|
||||||
|
|
||||||
final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
|
final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
|
||||||
|
@ -384,16 +372,8 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
|
||||||
final int codePointSetSize = 50;
|
final int codePointSetSize = 50;
|
||||||
final long seed = System.currentTimeMillis();
|
final long seed = System.currentTimeMillis();
|
||||||
final Random random = new Random(seed);
|
final Random random = new Random(seed);
|
||||||
|
final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
|
||||||
File dictFile = null;
|
final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
|
||||||
try {
|
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
setCurrentTimeForTestMode(mCurrentTime);
|
setCurrentTimeForTestMode(mCurrentTime);
|
||||||
final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
|
final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
|
||||||
|
|
||||||
|
@ -445,16 +425,9 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
|
||||||
final long seed = System.currentTimeMillis();
|
final long seed = System.currentTimeMillis();
|
||||||
final Random random = new Random(seed);
|
final Random random = new Random(seed);
|
||||||
|
|
||||||
File dictFile = null;
|
|
||||||
try {
|
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
setCurrentTimeForTestMode(mCurrentTime);
|
setCurrentTimeForTestMode(mCurrentTime);
|
||||||
|
final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
|
||||||
|
final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
|
||||||
|
|
||||||
final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
|
final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
|
||||||
final ArrayList<String> words = new ArrayList<>();
|
final ArrayList<String> words = new ArrayList<>();
|
||||||
|
@ -523,17 +496,9 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
|
||||||
final int codePointSetSize = 50;
|
final int codePointSetSize = 50;
|
||||||
final long seed = System.currentTimeMillis();
|
final long seed = System.currentTimeMillis();
|
||||||
final Random random = new Random(seed);
|
final Random random = new Random(seed);
|
||||||
|
|
||||||
File dictFile = null;
|
|
||||||
try {
|
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
setCurrentTimeForTestMode(mCurrentTime);
|
setCurrentTimeForTestMode(mCurrentTime);
|
||||||
|
final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
|
||||||
|
final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
|
||||||
final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
|
final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
|
||||||
|
|
||||||
final ArrayList<String> words = new ArrayList<>();
|
final ArrayList<String> words = new ArrayList<>();
|
||||||
|
@ -596,15 +561,8 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
|
||||||
|
|
||||||
private void testDictMigration(final int fromFormatVersion, final int toFormatVersion) {
|
private void testDictMigration(final int fromFormatVersion, final int toFormatVersion) {
|
||||||
setCurrentTimeForTestMode(mCurrentTime);
|
setCurrentTimeForTestMode(mCurrentTime);
|
||||||
File dictFile = null;
|
final File dictFile = createEmptyDictionaryAndGetFile(fromFormatVersion);
|
||||||
try {
|
final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", fromFormatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY);
|
addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY);
|
||||||
assertTrue(binaryDictionary.isValidWord("aaa"));
|
assertTrue(binaryDictionary.isValidWord("aaa"));
|
||||||
addUnigramWord(binaryDictionary, "bbb", Dictionary.NOT_A_PROBABILITY);
|
addUnigramWord(binaryDictionary, "bbb", Dictionary.NOT_A_PROBABILITY);
|
||||||
|
@ -634,7 +592,6 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
|
||||||
addBigramWords(binaryDictionary, "aaa", "bbb", Dictionary.NOT_A_PROBABILITY);
|
addBigramWords(binaryDictionary, "aaa", "bbb", Dictionary.NOT_A_PROBABILITY);
|
||||||
assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb"));
|
assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb"));
|
||||||
binaryDictionary.close();
|
binaryDictionary.close();
|
||||||
dictFile.delete();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testBeginningOfSentence() {
|
public void testBeginningOfSentence() {
|
||||||
|
@ -647,15 +604,8 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
|
||||||
|
|
||||||
private void testBeginningOfSentence(final int formatVersion) {
|
private void testBeginningOfSentence(final int formatVersion) {
|
||||||
setCurrentTimeForTestMode(mCurrentTime);
|
setCurrentTimeForTestMode(mCurrentTime);
|
||||||
File dictFile = null;
|
final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
|
||||||
try {
|
final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
|
|
||||||
binaryDictionary.addUnigramEntry("", DUMMY_PROBABILITY, "" /* shortcutTarget */,
|
binaryDictionary.addUnigramEntry("", DUMMY_PROBABILITY, "" /* shortcutTarget */,
|
||||||
BinaryDictionary.NOT_A_PROBABILITY /* shortcutProbability */,
|
BinaryDictionary.NOT_A_PROBABILITY /* shortcutProbability */,
|
||||||
|
@ -687,7 +637,6 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
|
||||||
assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa"));
|
assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa"));
|
||||||
assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb"));
|
assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb"));
|
||||||
binaryDictionary.close();
|
binaryDictionary.close();
|
||||||
dictFile.delete();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testRemoveUnigrams() {
|
public void testRemoveUnigrams() {
|
||||||
|
@ -699,15 +648,8 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
|
||||||
private void testRemoveUnigrams(final int formatVersion) {
|
private void testRemoveUnigrams(final int formatVersion) {
|
||||||
final int unigramInputCount = 20;
|
final int unigramInputCount = 20;
|
||||||
setCurrentTimeForTestMode(mCurrentTime);
|
setCurrentTimeForTestMode(mCurrentTime);
|
||||||
File dictFile = null;
|
final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
|
||||||
try {
|
final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
|
|
||||||
addUnigramWord(binaryDictionary, "aaa", Dictionary.NOT_A_PROBABILITY);
|
addUnigramWord(binaryDictionary, "aaa", Dictionary.NOT_A_PROBABILITY);
|
||||||
assertFalse(binaryDictionary.isValidWord("aaa"));
|
assertFalse(binaryDictionary.isValidWord("aaa"));
|
||||||
|
@ -719,6 +661,5 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
|
||||||
assertFalse(binaryDictionary.isValidWord("aaa"));
|
assertFalse(binaryDictionary.isValidWord("aaa"));
|
||||||
|
|
||||||
binaryDictionary.close();
|
binaryDictionary.close();
|
||||||
dictFile.delete();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -46,6 +46,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
private static final String TEST_LOCALE = "test";
|
private static final String TEST_LOCALE = "test";
|
||||||
private static final int[] DICT_FORMAT_VERSIONS =
|
private static final int[] DICT_FORMAT_VERSIONS =
|
||||||
new int[] { FormatSpec.VERSION4, FormatSpec.VERSION4_DEV };
|
new int[] { FormatSpec.VERSION4, FormatSpec.VERSION4_DEV };
|
||||||
|
private static final String DICTIONARY_ID = "TestBinaryDictionary";
|
||||||
|
|
||||||
private static boolean canCheckBigramProbability(final int formatVersion) {
|
private static boolean canCheckBigramProbability(final int formatVersion) {
|
||||||
return formatVersion > FormatSpec.VERSION401;
|
return formatVersion > FormatSpec.VERSION401;
|
||||||
|
@ -59,21 +60,42 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
return formatVersion >= FormatSpec.VERSION4_DEV;
|
return formatVersion >= FormatSpec.VERSION4_DEV;
|
||||||
}
|
}
|
||||||
|
|
||||||
private File createEmptyDictionaryAndGetFile(final String dictId,
|
private HashSet<File> mDictFilesToBeDeleted = new HashSet<>();
|
||||||
final int formatVersion) throws IOException {
|
|
||||||
|
@Override
|
||||||
|
protected void setUp() throws Exception {
|
||||||
|
super.setUp();
|
||||||
|
mDictFilesToBeDeleted.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void tearDown() throws Exception {
|
||||||
|
for (final File dictFile : mDictFilesToBeDeleted) {
|
||||||
|
dictFile.delete();
|
||||||
|
}
|
||||||
|
mDictFilesToBeDeleted.clear();
|
||||||
|
super.tearDown();
|
||||||
|
}
|
||||||
|
|
||||||
|
private File createEmptyDictionaryAndGetFile(final int formatVersion) {
|
||||||
if (formatVersion == FormatSpec.VERSION4
|
if (formatVersion == FormatSpec.VERSION4
|
||||||
|| formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING
|
|| formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING
|
||||||
|| formatVersion == FormatSpec.VERSION4_DEV) {
|
|| formatVersion == FormatSpec.VERSION4_DEV) {
|
||||||
return createEmptyVer4DictionaryAndGetFile(dictId, formatVersion);
|
try {
|
||||||
} else {
|
final File dictFile = createEmptyVer4DictionaryAndGetFile(formatVersion);
|
||||||
throw new IOException("Dictionary format version " + formatVersion
|
mDictFilesToBeDeleted.add(dictFile);
|
||||||
+ " is not supported.");
|
return dictFile;
|
||||||
|
} catch (final IOException e) {
|
||||||
|
fail(e.toString());
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
fail("Dictionary format version " + formatVersion + " is not supported.");
|
||||||
|
}
|
||||||
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
private File createEmptyVer4DictionaryAndGetFile(final String dictId,
|
private File createEmptyVer4DictionaryAndGetFile(final int formatVersion) throws IOException {
|
||||||
final int formatVersion) throws IOException {
|
final File file = File.createTempFile(DICTIONARY_ID, TEST_DICT_FILE_EXTENSION,
|
||||||
final File file = File.createTempFile(dictId, TEST_DICT_FILE_EXTENSION,
|
|
||||||
getContext().getCacheDir());
|
getContext().getCacheDir());
|
||||||
file.delete();
|
file.delete();
|
||||||
file.mkdir();
|
file.mkdir();
|
||||||
|
@ -87,6 +109,19 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private BinaryDictionary getBinaryDictionary(final File dictFile) {
|
||||||
|
return new BinaryDictionary(dictFile.getAbsolutePath(),
|
||||||
|
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
||||||
|
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
||||||
|
}
|
||||||
|
|
||||||
|
private BinaryDictionary getEmptyBinaryDictionary(final int formatVersion) {
|
||||||
|
final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
|
||||||
|
return new BinaryDictionary(dictFile.getAbsolutePath(),
|
||||||
|
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
||||||
|
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
||||||
|
}
|
||||||
|
|
||||||
public void testIsValidDictionary() {
|
public void testIsValidDictionary() {
|
||||||
for (final int formatVersion : DICT_FORMAT_VERSIONS) {
|
for (final int formatVersion : DICT_FORMAT_VERSIONS) {
|
||||||
testIsValidDictionary(formatVersion);
|
testIsValidDictionary(formatVersion);
|
||||||
|
@ -94,24 +129,15 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void testIsValidDictionary(final int formatVersion) {
|
private void testIsValidDictionary(final int formatVersion) {
|
||||||
File dictFile = null;
|
final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
|
||||||
try {
|
BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
assertTrue("binaryDictionary must be valid for existing valid dictionary file.",
|
assertTrue("binaryDictionary must be valid for existing valid dictionary file.",
|
||||||
binaryDictionary.isValidDictionary());
|
binaryDictionary.isValidDictionary());
|
||||||
binaryDictionary.close();
|
binaryDictionary.close();
|
||||||
assertFalse("binaryDictionary must be invalid after closing.",
|
assertFalse("binaryDictionary must be invalid after closing.",
|
||||||
binaryDictionary.isValidDictionary());
|
binaryDictionary.isValidDictionary());
|
||||||
FileUtils.deleteRecursively(dictFile);
|
FileUtils.deleteRecursively(dictFile);
|
||||||
binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */,
|
binaryDictionary = getBinaryDictionary(dictFile);
|
||||||
dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(),
|
|
||||||
TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
assertFalse("binaryDictionary must be invalid for not existing dictionary file.",
|
assertFalse("binaryDictionary must be invalid for not existing dictionary file.",
|
||||||
binaryDictionary.isValidDictionary());
|
binaryDictionary.isValidDictionary());
|
||||||
binaryDictionary.close();
|
binaryDictionary.close();
|
||||||
|
@ -124,15 +150,10 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void testConstructingDictionaryOnMemory(final int formatVersion) {
|
private void testConstructingDictionaryOnMemory(final int formatVersion) {
|
||||||
File dictFile = null;
|
final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
|
||||||
try {
|
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
FileUtils.deleteRecursively(dictFile);
|
FileUtils.deleteRecursively(dictFile);
|
||||||
assertFalse(dictFile.exists());
|
assertFalse(dictFile.exists());
|
||||||
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
||||||
true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, formatVersion,
|
true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, formatVersion,
|
||||||
new HashMap<String, String>());
|
new HashMap<String, String>());
|
||||||
assertTrue(binaryDictionary.isValidDictionary());
|
assertTrue(binaryDictionary.isValidDictionary());
|
||||||
|
@ -147,7 +168,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
assertEquals(formatVersion, binaryDictionary.getFormatVersion());
|
assertEquals(formatVersion, binaryDictionary.getFormatVersion());
|
||||||
assertEquals(probability, binaryDictionary.getFrequency("word"));
|
assertEquals(probability, binaryDictionary.getFrequency("word"));
|
||||||
binaryDictionary.close();
|
binaryDictionary.close();
|
||||||
dictFile.delete();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testAddTooLongWord() {
|
public void testAddTooLongWord() {
|
||||||
|
@ -157,16 +177,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void testAddTooLongWord(final int formatVersion) {
|
private void testAddTooLongWord(final int formatVersion) {
|
||||||
File dictFile = null;
|
final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion);
|
||||||
try {
|
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
|
|
||||||
final StringBuffer stringBuilder = new StringBuffer();
|
final StringBuffer stringBuilder = new StringBuffer();
|
||||||
for (int i = 0; i < Constants.DICTIONARY_MAX_WORD_LENGTH; i++) {
|
for (int i = 0; i < Constants.DICTIONARY_MAX_WORD_LENGTH; i++) {
|
||||||
stringBuilder.append('a');
|
stringBuilder.append('a');
|
||||||
|
@ -195,7 +206,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
assertEquals(BinaryDictionary.NOT_A_PROBABILITY,
|
assertEquals(BinaryDictionary.NOT_A_PROBABILITY,
|
||||||
binaryDictionary.getFrequency(invalidLongWord));
|
binaryDictionary.getFrequency(invalidLongWord));
|
||||||
assertEquals(updatedProbability, binaryDictionary.getFrequency("abc"));
|
assertEquals(updatedProbability, binaryDictionary.getFrequency("abc"));
|
||||||
dictFile.delete();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void addUnigramWord(final BinaryDictionary binaryDictionary, final String word,
|
private static void addUnigramWord(final BinaryDictionary binaryDictionary, final String word,
|
||||||
|
@ -256,16 +266,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void testAddUnigramWord(final int formatVersion) {
|
private void testAddUnigramWord(final int formatVersion) {
|
||||||
File dictFile = null;
|
final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion);
|
||||||
try {
|
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
|
|
||||||
final int probability = 100;
|
final int probability = 100;
|
||||||
addUnigramWord(binaryDictionary, "aaa", probability);
|
addUnigramWord(binaryDictionary, "aaa", probability);
|
||||||
// Reallocate and create.
|
// Reallocate and create.
|
||||||
|
@ -289,8 +290,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
assertEquals(probability, binaryDictionary.getFrequency("aaaa"));
|
assertEquals(probability, binaryDictionary.getFrequency("aaaa"));
|
||||||
assertEquals(probability, binaryDictionary.getFrequency("a"));
|
assertEquals(probability, binaryDictionary.getFrequency("a"));
|
||||||
assertEquals(updatedProbability, binaryDictionary.getFrequency("aaa"));
|
assertEquals(updatedProbability, binaryDictionary.getFrequency("aaa"));
|
||||||
|
|
||||||
dictFile.delete();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testRandomlyAddUnigramWord() {
|
public void testRandomlyAddUnigramWord() {
|
||||||
|
@ -303,16 +302,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
final int wordCount = 1000;
|
final int wordCount = 1000;
|
||||||
final int codePointSetSize = 50;
|
final int codePointSetSize = 50;
|
||||||
final long seed = System.currentTimeMillis();
|
final long seed = System.currentTimeMillis();
|
||||||
|
final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion);
|
||||||
File dictFile = null;
|
|
||||||
try {
|
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
|
|
||||||
final HashMap<String, Integer> probabilityMap = new HashMap<>();
|
final HashMap<String, Integer> probabilityMap = new HashMap<>();
|
||||||
// Test a word that isn't contained within the dictionary.
|
// Test a word that isn't contained within the dictionary.
|
||||||
|
@ -328,7 +318,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
for (String word : probabilityMap.keySet()) {
|
for (String word : probabilityMap.keySet()) {
|
||||||
assertEquals(word, (int)probabilityMap.get(word), binaryDictionary.getFrequency(word));
|
assertEquals(word, (int)probabilityMap.get(word), binaryDictionary.getFrequency(word));
|
||||||
}
|
}
|
||||||
dictFile.delete();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testAddBigramWords() {
|
public void testAddBigramWords() {
|
||||||
|
@ -338,15 +327,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void testAddBigramWords(final int formatVersion) {
|
private void testAddBigramWords(final int formatVersion) {
|
||||||
File dictFile = null;
|
final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion);
|
||||||
try {
|
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
|
|
||||||
final int unigramProbability = 100;
|
final int unigramProbability = 100;
|
||||||
final int bigramProbability = 150;
|
final int bigramProbability = 150;
|
||||||
|
@ -405,8 +386,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
assertEquals(updatedBigramProbability,
|
assertEquals(updatedBigramProbability,
|
||||||
getBigramProbability(binaryDictionary, "abcde", "fghij"));
|
getBigramProbability(binaryDictionary, "abcde", "fghij"));
|
||||||
}
|
}
|
||||||
|
|
||||||
dictFile.delete();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testRandomlyAddBigramWords() {
|
public void testRandomlyAddBigramWords() {
|
||||||
|
@ -421,16 +400,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
final int codePointSetSize = 50;
|
final int codePointSetSize = 50;
|
||||||
final long seed = System.currentTimeMillis();
|
final long seed = System.currentTimeMillis();
|
||||||
final Random random = new Random(seed);
|
final Random random = new Random(seed);
|
||||||
|
final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion);
|
||||||
File dictFile = null;
|
|
||||||
try {
|
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
|
|
||||||
final ArrayList<String> words = new ArrayList<>();
|
final ArrayList<String> words = new ArrayList<>();
|
||||||
final ArrayList<Pair<String, String>> bigramWords = new ArrayList<>();
|
final ArrayList<Pair<String, String>> bigramWords = new ArrayList<>();
|
||||||
|
@ -470,8 +440,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
getBigramProbability(binaryDictionary, bigram.first, bigram.second));
|
getBigramProbability(binaryDictionary, bigram.first, bigram.second));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
dictFile.delete();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testRemoveBigramWords() {
|
public void testRemoveBigramWords() {
|
||||||
|
@ -481,15 +449,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void testRemoveBigramWords(final int formatVersion) {
|
private void testRemoveBigramWords(final int formatVersion) {
|
||||||
File dictFile = null;
|
final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion);
|
||||||
try {
|
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
final int unigramProbability = 100;
|
final int unigramProbability = 100;
|
||||||
final int bigramProbability = 150;
|
final int bigramProbability = 150;
|
||||||
addUnigramWord(binaryDictionary, "aaa", unigramProbability);
|
addUnigramWord(binaryDictionary, "aaa", unigramProbability);
|
||||||
|
@ -522,8 +482,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
// Test remove non-existing bigram operation.
|
// Test remove non-existing bigram operation.
|
||||||
removeBigramEntry(binaryDictionary, "aaa", "abb");
|
removeBigramEntry(binaryDictionary, "aaa", "abb");
|
||||||
removeBigramEntry(binaryDictionary, "bcc", "aaa");
|
removeBigramEntry(binaryDictionary, "bcc", "aaa");
|
||||||
|
|
||||||
dictFile.delete();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testAddTrigramWords() {
|
public void testAddTrigramWords() {
|
||||||
|
@ -535,16 +493,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void testAddTrigramWords(final int formatVersion) {
|
private void testAddTrigramWords(final int formatVersion) {
|
||||||
File dictFile = null;
|
final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion);
|
||||||
try {
|
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
|
|
||||||
final int unigramProbability = 100;
|
final int unigramProbability = 100;
|
||||||
final int trigramProbability = 150;
|
final int trigramProbability = 150;
|
||||||
final int updatedTrigramProbability = 200;
|
final int updatedTrigramProbability = 200;
|
||||||
|
@ -572,8 +521,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
assertEquals(Dictionary.NOT_A_PROBABILITY,
|
assertEquals(Dictionary.NOT_A_PROBABILITY,
|
||||||
getTrigramProbability(binaryDictionary, "aaa", "abb", "bcc"));
|
getTrigramProbability(binaryDictionary, "aaa", "abb", "bcc"));
|
||||||
assertTrue(isValidBigram(binaryDictionary, "abb", "bcc"));
|
assertTrue(isValidBigram(binaryDictionary, "abb", "bcc"));
|
||||||
|
|
||||||
dictFile.delete();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testFlushDictionary() {
|
public void testFlushDictionary() {
|
||||||
|
@ -583,15 +530,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void testFlushDictionary(final int formatVersion) {
|
private void testFlushDictionary(final int formatVersion) {
|
||||||
File dictFile = null;
|
final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
|
||||||
try {
|
BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
|
|
||||||
final int probability = 100;
|
final int probability = 100;
|
||||||
addUnigramWord(binaryDictionary, "aaa", probability);
|
addUnigramWord(binaryDictionary, "aaa", probability);
|
||||||
|
@ -611,23 +551,16 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
binaryDictionary.flush();
|
binaryDictionary.flush();
|
||||||
binaryDictionary.close();
|
binaryDictionary.close();
|
||||||
|
|
||||||
binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
binaryDictionary = getBinaryDictionary(dictFile);
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
|
|
||||||
assertEquals(probability, binaryDictionary.getFrequency("aaa"));
|
assertEquals(probability, binaryDictionary.getFrequency("aaa"));
|
||||||
assertEquals(probability, binaryDictionary.getFrequency("abcd"));
|
assertEquals(probability, binaryDictionary.getFrequency("abcd"));
|
||||||
addUnigramWord(binaryDictionary, "bcde", probability);
|
addUnigramWord(binaryDictionary, "bcde", probability);
|
||||||
binaryDictionary.flush();
|
binaryDictionary.flush();
|
||||||
binaryDictionary.close();
|
binaryDictionary.close();
|
||||||
|
|
||||||
binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
binaryDictionary = getBinaryDictionary(dictFile);
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
assertEquals(probability, binaryDictionary.getFrequency("bcde"));
|
assertEquals(probability, binaryDictionary.getFrequency("bcde"));
|
||||||
binaryDictionary.close();
|
binaryDictionary.close();
|
||||||
|
|
||||||
dictFile.delete();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testFlushWithGCDictionary() {
|
public void testFlushWithGCDictionary() {
|
||||||
|
@ -637,16 +570,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void testFlushWithGCDictionary(final int formatVersion) {
|
private void testFlushWithGCDictionary(final int formatVersion) {
|
||||||
File dictFile = null;
|
final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
|
||||||
try {
|
BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
|
|
||||||
final int unigramProbability = 100;
|
final int unigramProbability = 100;
|
||||||
final int bigramProbability = 150;
|
final int bigramProbability = 150;
|
||||||
addUnigramWord(binaryDictionary, "aaa", unigramProbability);
|
addUnigramWord(binaryDictionary, "aaa", unigramProbability);
|
||||||
|
@ -659,9 +584,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
binaryDictionary.flushWithGC();
|
binaryDictionary.flushWithGC();
|
||||||
binaryDictionary.close();
|
binaryDictionary.close();
|
||||||
|
|
||||||
binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
binaryDictionary = getBinaryDictionary(dictFile);
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa"));
|
assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa"));
|
||||||
assertEquals(unigramProbability, binaryDictionary.getFrequency("abb"));
|
assertEquals(unigramProbability, binaryDictionary.getFrequency("abb"));
|
||||||
assertEquals(unigramProbability, binaryDictionary.getFrequency("bcc"));
|
assertEquals(unigramProbability, binaryDictionary.getFrequency("bcc"));
|
||||||
|
@ -676,8 +599,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
assertFalse(isValidBigram(binaryDictionary, "aaa", "aaa"));
|
assertFalse(isValidBigram(binaryDictionary, "aaa", "aaa"));
|
||||||
binaryDictionary.flushWithGC();
|
binaryDictionary.flushWithGC();
|
||||||
binaryDictionary.close();
|
binaryDictionary.close();
|
||||||
|
|
||||||
dictFile.delete();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testAddBigramWordsAndFlashWithGC() {
|
public void testAddBigramWordsAndFlashWithGC() {
|
||||||
|
@ -694,16 +615,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
final long seed = System.currentTimeMillis();
|
final long seed = System.currentTimeMillis();
|
||||||
final Random random = new Random(seed);
|
final Random random = new Random(seed);
|
||||||
|
|
||||||
File dictFile = null;
|
final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
|
||||||
try {
|
BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
|
|
||||||
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
|
|
||||||
final ArrayList<String> words = new ArrayList<>();
|
final ArrayList<String> words = new ArrayList<>();
|
||||||
final ArrayList<Pair<String, String>> bigramWords = new ArrayList<>();
|
final ArrayList<Pair<String, String>> bigramWords = new ArrayList<>();
|
||||||
|
@ -736,10 +649,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
|
|
||||||
binaryDictionary.flushWithGC();
|
binaryDictionary.flushWithGC();
|
||||||
binaryDictionary.close();
|
binaryDictionary.close();
|
||||||
binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
binaryDictionary = getBinaryDictionary(dictFile);
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
|
|
||||||
|
|
||||||
for (final Pair<String, String> bigram : bigramWords) {
|
for (final Pair<String, String> bigram : bigramWords) {
|
||||||
final int bigramProbability = bigramProbabilities.get(bigram);
|
final int bigramProbability = bigramProbabilities.get(bigram);
|
||||||
|
@ -750,8 +660,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
getBigramProbability(binaryDictionary, bigram.first, bigram.second));
|
getBigramProbability(binaryDictionary, bigram.first, bigram.second));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
dictFile.delete();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testRandomOperationsAndFlashWithGC() {
|
public void testRandomOperationsAndFlashWithGC() {
|
||||||
|
@ -771,17 +679,9 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
|
|
||||||
final long seed = System.currentTimeMillis();
|
final long seed = System.currentTimeMillis();
|
||||||
final Random random = new Random(seed);
|
final Random random = new Random(seed);
|
||||||
|
final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
|
||||||
|
BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
|
||||||
|
|
||||||
File dictFile = null;
|
|
||||||
try {
|
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
|
|
||||||
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
final ArrayList<String> words = new ArrayList<>();
|
final ArrayList<String> words = new ArrayList<>();
|
||||||
final ArrayList<Pair<String, String>> bigramWords = new ArrayList<>();
|
final ArrayList<Pair<String, String>> bigramWords = new ArrayList<>();
|
||||||
final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
|
final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
|
||||||
|
@ -798,9 +698,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
binaryDictionary.close();
|
binaryDictionary.close();
|
||||||
|
|
||||||
for (int gcCount = 0; gcCount < flashWithGCIterationCount; gcCount++) {
|
for (int gcCount = 0; gcCount < flashWithGCIterationCount; gcCount++) {
|
||||||
binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
binaryDictionary = getBinaryDictionary(dictFile);
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
for (int opCount = 0; opCount < operationCountInEachIteration; opCount++) {
|
for (int opCount = 0; opCount < operationCountInEachIteration; opCount++) {
|
||||||
// Add unigram.
|
// Add unigram.
|
||||||
if (random.nextFloat() < addUnigramProb) {
|
if (random.nextFloat() < addUnigramProb) {
|
||||||
|
@ -867,8 +765,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
binaryDictionary.flushWithGC();
|
binaryDictionary.flushWithGC();
|
||||||
binaryDictionary.close();
|
binaryDictionary.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
dictFile.delete();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testAddManyUnigramsAndFlushWithGC() {
|
public void testAddManyUnigramsAndFlushWithGC() {
|
||||||
|
@ -884,12 +780,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
final long seed = System.currentTimeMillis();
|
final long seed = System.currentTimeMillis();
|
||||||
final Random random = new Random(seed);
|
final Random random = new Random(seed);
|
||||||
|
|
||||||
File dictFile = null;
|
final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
|
||||||
try {
|
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
|
|
||||||
final ArrayList<String> words = new ArrayList<>();
|
final ArrayList<String> words = new ArrayList<>();
|
||||||
final HashMap<String, Integer> unigramProbabilities = new HashMap<>();
|
final HashMap<String, Integer> unigramProbabilities = new HashMap<>();
|
||||||
|
@ -897,9 +788,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
|
|
||||||
BinaryDictionary binaryDictionary;
|
BinaryDictionary binaryDictionary;
|
||||||
for (int i = 0; i < flashWithGCIterationCount; i++) {
|
for (int i = 0; i < flashWithGCIterationCount; i++) {
|
||||||
binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
binaryDictionary = getBinaryDictionary(dictFile);
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
while(!binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
|
while(!binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
|
||||||
final String word = CodePointUtils.generateWord(random, codePointSet);
|
final String word = CodePointUtils.generateWord(random, codePointSet);
|
||||||
words.add(word);
|
words.add(word);
|
||||||
|
@ -917,8 +806,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
binaryDictionary.flushWithGC();
|
binaryDictionary.flushWithGC();
|
||||||
binaryDictionary.close();
|
binaryDictionary.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
dictFile.delete();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testUnigramAndBigramCount() {
|
public void testUnigramAndBigramCount() {
|
||||||
|
@ -934,13 +821,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
final int bigramCountPerIteration = 2000;
|
final int bigramCountPerIteration = 2000;
|
||||||
final long seed = System.currentTimeMillis();
|
final long seed = System.currentTimeMillis();
|
||||||
final Random random = new Random(seed);
|
final Random random = new Random(seed);
|
||||||
|
final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
|
||||||
File dictFile = null;
|
|
||||||
try {
|
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
|
|
||||||
final ArrayList<String> words = new ArrayList<>();
|
final ArrayList<String> words = new ArrayList<>();
|
||||||
final HashSet<Pair<String, String>> bigrams = new HashSet<>();
|
final HashSet<Pair<String, String>> bigrams = new HashSet<>();
|
||||||
|
@ -948,9 +829,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
|
|
||||||
BinaryDictionary binaryDictionary;
|
BinaryDictionary binaryDictionary;
|
||||||
for (int i = 0; i < flashWithGCIterationCount; i++) {
|
for (int i = 0; i < flashWithGCIterationCount; i++) {
|
||||||
binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
binaryDictionary = getBinaryDictionary(dictFile);
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
for (int j = 0; j < unigramCountPerIteration; j++) {
|
for (int j = 0; j < unigramCountPerIteration; j++) {
|
||||||
final String word = CodePointUtils.generateWord(random, codePointSet);
|
final String word = CodePointUtils.generateWord(random, codePointSet);
|
||||||
words.add(word);
|
words.add(word);
|
||||||
|
@ -982,8 +861,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
BinaryDictionary.BIGRAM_COUNT_QUERY)));
|
BinaryDictionary.BIGRAM_COUNT_QUERY)));
|
||||||
binaryDictionary.close();
|
binaryDictionary.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
dictFile.delete();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testAddMultipleDictionaryEntries() {
|
public void testAddMultipleDictionaryEntries() {
|
||||||
|
@ -998,13 +875,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
final double bigramContinueRate = 0.9;
|
final double bigramContinueRate = 0.9;
|
||||||
final long seed = System.currentTimeMillis();
|
final long seed = System.currentTimeMillis();
|
||||||
final Random random = new Random(seed);
|
final Random random = new Random(seed);
|
||||||
|
final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
|
||||||
File dictFile = null;
|
|
||||||
try {
|
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
|
|
||||||
final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
|
final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
|
||||||
final HashMap<String, Integer> unigramProbabilities = new HashMap<>();
|
final HashMap<String, Integer> unigramProbabilities = new HashMap<>();
|
||||||
|
@ -1029,9 +900,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
prevWord = (random.nextDouble() < bigramContinueRate) ? word : null;
|
prevWord = (random.nextDouble() < bigramContinueRate) ? word : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
binaryDictionary.addMultipleDictionaryEntries(languageModelParams);
|
binaryDictionary.addMultipleDictionaryEntries(languageModelParams);
|
||||||
|
|
||||||
for (Map.Entry<String, Integer> entry : unigramProbabilities.entrySet()) {
|
for (Map.Entry<String, Integer> entry : unigramProbabilities.entrySet()) {
|
||||||
|
@ -1064,16 +933,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
final int BIGRAM_COUNT = 1000;
|
final int BIGRAM_COUNT = 1000;
|
||||||
final int codePointSetSize = 20;
|
final int codePointSetSize = 20;
|
||||||
final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
|
final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
|
||||||
|
final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
|
||||||
File dictFile = null;
|
final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
|
||||||
try {
|
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
|
|
||||||
final WordProperty invalidWordProperty = binaryDictionary.getWordProperty("dummyWord",
|
final WordProperty invalidWordProperty = binaryDictionary.getWordProperty("dummyWord",
|
||||||
false /* isBeginningOfSentence */);
|
false /* isBeginningOfSentence */);
|
||||||
|
@ -1168,16 +1029,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
final int BIGRAM_COUNT = 1000;
|
final int BIGRAM_COUNT = 1000;
|
||||||
final int codePointSetSize = 20;
|
final int codePointSetSize = 20;
|
||||||
final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
|
final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
|
||||||
|
final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion);
|
||||||
File dictFile = null;
|
|
||||||
try {
|
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
|
|
||||||
final WordProperty invalidWordProperty = binaryDictionary.getWordProperty("dummyWord",
|
final WordProperty invalidWordProperty = binaryDictionary.getWordProperty("dummyWord",
|
||||||
false /* isBeginningOfSentence */);
|
false /* isBeginningOfSentence */);
|
||||||
|
@ -1262,15 +1114,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void testAddShortcuts(final int formatVersion) {
|
private void testAddShortcuts(final int formatVersion) {
|
||||||
File dictFile = null;
|
final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion);
|
||||||
try {
|
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
|
|
||||||
final int unigramProbability = 100;
|
final int unigramProbability = 100;
|
||||||
final int shortcutProbability = 10;
|
final int shortcutProbability = 10;
|
||||||
|
@ -1338,16 +1182,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
final ArrayList<String> words = new ArrayList<>();
|
final ArrayList<String> words = new ArrayList<>();
|
||||||
final HashMap<String, Integer> unigramProbabilities = new HashMap<>();
|
final HashMap<String, Integer> unigramProbabilities = new HashMap<>();
|
||||||
final HashMap<String, HashMap<String, Integer>> shortcutTargets = new HashMap<>();
|
final HashMap<String, HashMap<String, Integer>> shortcutTargets = new HashMap<>();
|
||||||
|
final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion);
|
||||||
File dictFile = null;
|
|
||||||
try {
|
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
|
|
||||||
for (int i = 0; i < UNIGRAM_COUNT; i++) {
|
for (int i = 0; i < UNIGRAM_COUNT; i++) {
|
||||||
final String word = CodePointUtils.generateWord(random, codePointSet);
|
final String word = CodePointUtils.generateWord(random, codePointSet);
|
||||||
|
@ -1405,15 +1240,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void testDictMigration(final int fromFormatVersion, final int toFormatVersion) {
|
private void testDictMigration(final int fromFormatVersion, final int toFormatVersion) {
|
||||||
File dictFile = null;
|
final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(fromFormatVersion);
|
||||||
try {
|
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", fromFormatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
final int unigramProbability = 100;
|
final int unigramProbability = 100;
|
||||||
addUnigramWord(binaryDictionary, "aaa", unigramProbability);
|
addUnigramWord(binaryDictionary, "aaa", unigramProbability);
|
||||||
addUnigramWord(binaryDictionary, "bbb", unigramProbability);
|
addUnigramWord(binaryDictionary, "bbb", unigramProbability);
|
||||||
|
@ -1465,16 +1292,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
final int codePointSetSize = 50;
|
final int codePointSetSize = 50;
|
||||||
final long seed = System.currentTimeMillis();
|
final long seed = System.currentTimeMillis();
|
||||||
final Random random = new Random(seed);
|
final Random random = new Random(seed);
|
||||||
|
final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(fromFormatVersion);
|
||||||
File dictFile = null;
|
|
||||||
try {
|
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", fromFormatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
|
|
||||||
final ArrayList<String> words = new ArrayList<>();
|
final ArrayList<String> words = new ArrayList<>();
|
||||||
final ArrayList<Pair<String, String>> bigrams = new ArrayList<>();
|
final ArrayList<Pair<String, String>> bigrams = new ArrayList<>();
|
||||||
|
@ -1540,15 +1358,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void testBeginningOfSentence(final int formatVersion) {
|
private void testBeginningOfSentence(final int formatVersion) {
|
||||||
File dictFile = null;
|
final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion);
|
||||||
try {
|
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
final int dummyProbability = 0;
|
final int dummyProbability = 0;
|
||||||
final NgramContext beginningOfSentenceContext = NgramContext.BEGINNING_OF_SENTENCE;
|
final NgramContext beginningOfSentenceContext = NgramContext.BEGINNING_OF_SENTENCE;
|
||||||
final int bigramProbability = 200;
|
final int bigramProbability = 200;
|
||||||
|
@ -1576,15 +1386,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void testGetMaxFrequencyOfExactMatches(final int formatVersion) {
|
private void testGetMaxFrequencyOfExactMatches(final int formatVersion) {
|
||||||
File dictFile = null;
|
final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion);
|
||||||
try {
|
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
|
|
||||||
} catch (IOException e) {
|
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
|
||||||
}
|
|
||||||
final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
|
||||||
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
|
|
||||||
addUnigramWord(binaryDictionary, "abc", 10);
|
addUnigramWord(binaryDictionary, "abc", 10);
|
||||||
addUnigramWord(binaryDictionary, "aBc", 15);
|
addUnigramWord(binaryDictionary, "aBc", 15);
|
||||||
assertEquals(15, binaryDictionary.getMaxFrequencyOfExactMatches("abc"));
|
assertEquals(15, binaryDictionary.getMaxFrequencyOfExactMatches("abc"));
|
||||||
|
|
Loading…
Reference in New Issue