nyancrimew/LatinIME
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge "Add a dedicated method to control time in native code."

Browse source
This commit is contained in:
Keisuke Kuroyanagi 2014-01-16 17:25:28 +00:00 committed by Android (Google) Code Review
commit f4bf620502
6 changed files with 69 additions and 82 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
java/src/com/android/inputmethod/latin/BinaryDictionary.java
View file

@ -165,6 +165,7 @@ public final class BinaryDictionary extends Dictionary {
LanguageModelParam[] languageModelParams, int startIndex);
private static native int calculateProbabilityNative(long dict, int unigramProbability,
int bigramProbability);
private static native int setCurrentTimeForTestNative(int currentTime);
private static native String getPropertyNative(long dict, String query);
@UsedForTesting
@ -420,8 +421,22 @@ public final class BinaryDictionary extends Dictionary {
return calculateProbabilityNative(mNativeDict, unigramProbability, bigramProbability);
}
/**
* Control the current time to be used in the native code. If currentTime >= 0, this method sets
* the current time and gets into test mode.
* In test mode, set timestamp is used as the current time in the native code.
* If currentTime < 0, quit the test mode and returns to using time() to get the current time.
*
* @param currentTime seconds since the unix epoch
* @return current time got in the native code.
*/
@UsedForTesting
public String getPropertyForTests(String query) {
public static int setCurrentTimeForTest(final int currentTime) {
return setCurrentTimeForTestNative(currentTime);
}
@UsedForTesting
public String getPropertyForTest(final String query) {
if (!isValidDictionary()) return "";
return getPropertyNative(mNativeDict, query);
}

17
native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
View file

@ -29,6 +29,7 @@
#include "suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h"
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
#include "utils/autocorrection_threshold_utils.h"
#include "utils/time_keeper.h"
namespace latinime {
@ -457,6 +458,17 @@ static jstring latinime_BinaryDictionary_getProperty(JNIEnv *env, jclass clazz,
return env->NewStringUTF(resultChars);
}
static int latinime_BinaryDictionary_setCurrentTimeForTest(JNIEnv *env, jclass clazz,
jint currentTime) {
if (currentTime >= 0) {
TimeKeeper::startTestModeWithForceCurrentTime(currentTime);
} else {
TimeKeeper::stopTestMode();
}
TimeKeeper::setCurrentTime();
return TimeKeeper::peekCurrentTime();
}
static const JNINativeMethod sMethods[] = {
{
const_cast<char *>("createEmptyDictFileNative"),
@ -549,6 +561,11 @@ static const JNINativeMethod sMethods[] = {
const_cast<char *>("(JII)I"),
reinterpret_cast<void *>(latinime_BinaryDictionary_calculateProbabilityNative)
},
{
const_cast<char *>("setCurrentTimeForTestNative"),
const_cast<char *>("(I)I"),
reinterpret_cast<void *>(latinime_BinaryDictionary_setCurrentTimeForTest)
},
{
const_cast<char *>("getPropertyNative"),
const_cast<char *>("(JLjava/lang/String;)Ljava/lang/String;"),

12
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
View file

@ -34,11 +34,6 @@ const char *const Ver4PatriciaTriePolicy::UNIGRAM_COUNT_QUERY = "UNIGRAM_COUNT";
const char *const Ver4PatriciaTriePolicy::BIGRAM_COUNT_QUERY = "BIGRAM_COUNT";
const char *const Ver4PatriciaTriePolicy::MAX_UNIGRAM_COUNT_QUERY = "MAX_UNIGRAM_COUNT";
const char *const Ver4PatriciaTriePolicy::MAX_BIGRAM_COUNT_QUERY = "MAX_BIGRAM_COUNT";
const char *const Ver4PatriciaTriePolicy::SET_CURRENT_TIME_FOR_TESTING_QUERY_FORMAT =
"SET_CURRENT_TIME_FOR_TESTING:%d";
const char *const Ver4PatriciaTriePolicy::GET_CURRENT_TIME_QUERY = "GET_CURRENT_TIME";
const char *const Ver4PatriciaTriePolicy::QUIT_TIMEKEEPER_TEST_MODE_QUERY =
"QUIT_TIMEKEEPER_TEST_MODE";
const int Ver4PatriciaTriePolicy::MARGIN_TO_REFUSE_DYNAMIC_OPERATIONS = 1024;
const int Ver4PatriciaTriePolicy::MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS =
Ver4DictConstants::MAX_DICTIONARY_SIZE - MARGIN_TO_REFUSE_DYNAMIC_OPERATIONS;
@ -288,7 +283,6 @@ bool Ver4PatriciaTriePolicy::needsToRunGC(const bool mindsBlockByGC) const {
void Ver4PatriciaTriePolicy::getProperty(const char *const query, const int queryLength,
char *const outResult, const int maxResultLength) {
const int compareLength = queryLength + 1 /* terminator */;
int timestamp = NOT_A_TIMESTAMP;
if (strncmp(query, UNIGRAM_COUNT_QUERY, compareLength) == 0) {
snprintf(outResult, maxResultLength, "%d", mUnigramCount);
} else if (strncmp(query, BIGRAM_COUNT_QUERY, compareLength) == 0) {
@ -301,12 +295,6 @@ void Ver4PatriciaTriePolicy::getProperty(const char *const query, const int quer
snprintf(outResult, maxResultLength, "%d",
mHeaderPolicy->isDecayingDict() ? ForgettingCurveUtils::MAX_BIGRAM_COUNT :
static_cast<int>(Ver4DictConstants::MAX_DICTIONARY_SIZE));
} else if (sscanf(query, SET_CURRENT_TIME_FOR_TESTING_QUERY_FORMAT, &timestamp) == 1) {
TimeKeeper::startTestModeWithForceCurrentTime(timestamp);
} else if (strncmp(query, GET_CURRENT_TIME_QUERY, compareLength) == 0) {
snprintf(outResult, maxResultLength, "%d", TimeKeeper::peekCurrentTime());
} else if (strncmp(query, QUIT_TIMEKEEPER_TEST_MODE_QUERY, compareLength) == 0) {
TimeKeeper::stopTestMode();
}
}

3
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
View file

@ -116,9 +116,6 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
static const char *const BIGRAM_COUNT_QUERY;
static const char *const MAX_UNIGRAM_COUNT_QUERY;
static const char *const MAX_BIGRAM_COUNT_QUERY;
static const char *const SET_CURRENT_TIME_FOR_TESTING_QUERY_FORMAT;
static const char *const GET_CURRENT_TIME_QUERY;
static const char *const QUIT_TIMEKEEPER_TEST_MODE_QUERY;
// When the dictionary size is near the maximum size, we have to refuse dynamic operations to
// prevent the dictionary from overflowing.
static const int MARGIN_TO_REFUSE_DYNAMIC_OPERATIONS;

94
tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
View file

@ -37,14 +37,6 @@ import java.util.concurrent.TimeUnit;
public class BinaryDictionaryDecayingTests extends AndroidTestCase {
private static final String TEST_DICT_FILE_EXTENSION = ".testDict";
private static final String TEST_LOCALE = "test";
// Note that these are corresponding definitions in native code in
// latinime::Ver4PatriciaTriePolicy.
private static final String SET_CURRENT_TIME_FOR_TESTING_QUERY =
"SET_CURRENT_TIME_FOR_TESTING";
private static final String GET_CURRENT_TIME_QUERY = "GET_CURRENT_TIME";
private static final String QUIT_TIMEKEEPER_TEST_MODE_QUERY = "QUIT_TIMEKEEPER_TEST_MODE";
private static final int DUMMY_PROBABILITY = 0;
private int mCurrentTime = 0;
@ -58,17 +50,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
@Override
protected void tearDown() throws Exception {
super.tearDown();
try {
final File dictFile =
createEmptyDictionaryAndGetFile("TestBinaryDictionary", FormatSpec.VERSION4);
final BinaryDictionary binaryDictionary =
new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */,
dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(),
TEST_LOCALE, true /* isUpdatable */);
binaryDictionary.getPropertyForTests(QUIT_TIMEKEEPER_TEST_MODE_QUERY);
} catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e);
}
stopTestModeInNativeCode();
}
private void addUnigramWord(final BinaryDictionary binaryDictionary, final String word,
@ -89,7 +71,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
// 4 days.
final int timeToElapse = (int)TimeUnit.SECONDS.convert(4, TimeUnit.DAYS);
mCurrentTime += timeToElapse;
setCurrentTime(binaryDictionary, mCurrentTime);
setCurrentTimeForTestMode(mCurrentTime);
binaryDictionary.flushWithGC();
}
@ -97,7 +79,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
// 60 days.
final int timeToElapse = (int)TimeUnit.SECONDS.convert(60, TimeUnit.DAYS);
mCurrentTime += timeToElapse;
setCurrentTime(binaryDictionary, mCurrentTime);
setCurrentTimeForTestMode(mCurrentTime);
binaryDictionary.flushWithGC();
}
@ -110,6 +92,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
+ " is not supported.");
}
}
private File createEmptyVer4DictionaryAndGetFile(final String dictId) throws IOException {
final File file = File.createTempFile(dictId, TEST_DICT_FILE_EXTENSION,
getContext().getCacheDir());
@ -128,14 +111,12 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
}
}
private static int getCurrentTime(final BinaryDictionary binaryDictionary) {
return Integer.parseInt(binaryDictionary.getPropertyForTests(GET_CURRENT_TIME_QUERY));
private static int setCurrentTimeForTestMode(final int currentTime) {
return BinaryDictionary.setCurrentTimeForTest(currentTime);
}
private static void setCurrentTime(final BinaryDictionary binaryDictionary,
final int currentTime) {
final String query = SET_CURRENT_TIME_FOR_TESTING_QUERY + ":" + currentTime;
binaryDictionary.getPropertyForTests(query);
private static int stopTestModeInNativeCode() {
return BinaryDictionary.setCurrentTimeForTest(-1);
}
public void testControlCurrentTime() {
@ -146,24 +127,13 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
final int TEST_COUNT = 1000;
final long seed = System.currentTimeMillis();
final Random random = new Random(seed);
File dictFile = null;
try {
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
} catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e);
}
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
binaryDictionary.getPropertyForTests(QUIT_TIMEKEEPER_TEST_MODE_QUERY);
final int startTime = getCurrentTime(binaryDictionary);
final int startTime = stopTestModeInNativeCode();
for (int i = 0; i < TEST_COUNT; i++) {
final int currentTime = random.nextInt(Integer.MAX_VALUE);
setCurrentTime(binaryDictionary, currentTime);
assertEquals(currentTime, getCurrentTime(binaryDictionary));
final int currentTimeInNativeCode = setCurrentTimeForTestMode(currentTime);
assertEquals(currentTime, currentTimeInNativeCode);
}
binaryDictionary.getPropertyForTests(QUIT_TIMEKEEPER_TEST_MODE_QUERY);
final int endTime = getCurrentTime(binaryDictionary);
final int endTime = stopTestModeInNativeCode();
final int MAX_ALLOWED_ELAPSED_TIME = 10;
assertTrue(startTime <= endTime && endTime <= startTime + MAX_ALLOWED_ELAPSED_TIME);
}
@ -301,7 +271,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
setCurrentTime(binaryDictionary, mCurrentTime);
setCurrentTimeForTestMode(mCurrentTime);
final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
final ArrayList<String> words = new ArrayList<String>();
@ -312,31 +282,31 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
}
final int maxUnigramCount = Integer.parseInt(
binaryDictionary.getPropertyForTests(BinaryDictionary.MAX_UNIGRAM_COUNT_QUERY));
binaryDictionary.getPropertyForTest(BinaryDictionary.MAX_UNIGRAM_COUNT_QUERY));
for (int i = 0; i < unigramTypedCount; i++) {
final String word = words.get(random.nextInt(words.size()));
addUnigramWord(binaryDictionary, word, DUMMY_PROBABILITY);
if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
final int unigramCountBeforeGC =
Integer.parseInt(binaryDictionary.getPropertyForTests(
Integer.parseInt(binaryDictionary.getPropertyForTest(
BinaryDictionary.UNIGRAM_COUNT_QUERY));
while (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
forcePassingShortTime(binaryDictionary);
}
final int unigramCountAfterGC =
Integer.parseInt(binaryDictionary.getPropertyForTests(
Integer.parseInt(binaryDictionary.getPropertyForTest(
BinaryDictionary.UNIGRAM_COUNT_QUERY));
assertTrue(unigramCountBeforeGC > unigramCountAfterGC);
}
}
assertTrue(Integer.parseInt(binaryDictionary.getPropertyForTests(
assertTrue(Integer.parseInt(binaryDictionary.getPropertyForTest(
BinaryDictionary.UNIGRAM_COUNT_QUERY)) > 0);
assertTrue(Integer.parseInt(binaryDictionary.getPropertyForTests(
assertTrue(Integer.parseInt(binaryDictionary.getPropertyForTest(
BinaryDictionary.UNIGRAM_COUNT_QUERY)) <= maxUnigramCount);
forcePassingLongTime(binaryDictionary);
assertEquals(0, Integer.parseInt(binaryDictionary.getPropertyForTests(
assertEquals(0, Integer.parseInt(binaryDictionary.getPropertyForTest(
BinaryDictionary.UNIGRAM_COUNT_QUERY)));
}
@ -362,7 +332,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
setCurrentTime(binaryDictionary, mCurrentTime);
setCurrentTimeForTestMode(mCurrentTime);
final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
final String strong = "strong";
@ -383,13 +353,13 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
}
if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
final int unigramCountBeforeGC =
Integer.parseInt(binaryDictionary.getPropertyForTests(
Integer.parseInt(binaryDictionary.getPropertyForTest(
BinaryDictionary.UNIGRAM_COUNT_QUERY));
assertTrue(binaryDictionary.isValidWord(strong));
assertTrue(binaryDictionary.isValidWord(weak));
binaryDictionary.flushWithGC();
final int unigramCountAfterGC =
Integer.parseInt(binaryDictionary.getPropertyForTests(
Integer.parseInt(binaryDictionary.getPropertyForTest(
BinaryDictionary.UNIGRAM_COUNT_QUERY));
assertTrue(unigramCountBeforeGC > unigramCountAfterGC);
assertFalse(binaryDictionary.isValidWord(weak));
@ -420,7 +390,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
setCurrentTime(binaryDictionary, mCurrentTime);
setCurrentTimeForTestMode(mCurrentTime);
final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
final ArrayList<String> words = new ArrayList<String>();
@ -443,7 +413,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
}
final int maxBigramCount = Integer.parseInt(
binaryDictionary.getPropertyForTests(BinaryDictionary.MAX_BIGRAM_COUNT_QUERY));
binaryDictionary.getPropertyForTest(BinaryDictionary.MAX_BIGRAM_COUNT_QUERY));
for (int i = 0; i < bigramTypedCount; ++i) {
final Pair<String, String> bigram = bigrams.get(random.nextInt(bigrams.size()));
addUnigramWord(binaryDictionary, bigram.first, DUMMY_PROBABILITY);
@ -452,24 +422,24 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
final int bigramCountBeforeGC =
Integer.parseInt(binaryDictionary.getPropertyForTests(
Integer.parseInt(binaryDictionary.getPropertyForTest(
BinaryDictionary.BIGRAM_COUNT_QUERY));
while (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
forcePassingShortTime(binaryDictionary);
}
final int bigramCountAfterGC =
Integer.parseInt(binaryDictionary.getPropertyForTests(
Integer.parseInt(binaryDictionary.getPropertyForTest(
BinaryDictionary.BIGRAM_COUNT_QUERY));
assertTrue(bigramCountBeforeGC > bigramCountAfterGC);
}
}
assertTrue(Integer.parseInt(binaryDictionary.getPropertyForTests(
assertTrue(Integer.parseInt(binaryDictionary.getPropertyForTest(
BinaryDictionary.BIGRAM_COUNT_QUERY)) > 0);
assertTrue(Integer.parseInt(binaryDictionary.getPropertyForTests(
assertTrue(Integer.parseInt(binaryDictionary.getPropertyForTest(
BinaryDictionary.BIGRAM_COUNT_QUERY)) <= maxBigramCount);
forcePassingLongTime(binaryDictionary);
assertEquals(0, Integer.parseInt(binaryDictionary.getPropertyForTests(
assertEquals(0, Integer.parseInt(binaryDictionary.getPropertyForTest(
BinaryDictionary.BIGRAM_COUNT_QUERY)));
}
@ -497,7 +467,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
setCurrentTime(binaryDictionary, mCurrentTime);
setCurrentTimeForTestMode(mCurrentTime);
final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
final ArrayList<String> words = new ArrayList<String>();
@ -538,11 +508,11 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
}
if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
final int bigramCountBeforeGC =
Integer.parseInt(binaryDictionary.getPropertyForTests(
Integer.parseInt(binaryDictionary.getPropertyForTest(
BinaryDictionary.BIGRAM_COUNT_QUERY));
binaryDictionary.flushWithGC();
final int bigramCountAfterGC =
Integer.parseInt(binaryDictionary.getPropertyForTests(
Integer.parseInt(binaryDictionary.getPropertyForTest(
BinaryDictionary.BIGRAM_COUNT_QUERY));
assertTrue(bigramCountBeforeGC > bigramCountAfterGC);
assertTrue(binaryDictionary.isValidBigram(strong, target));

8
tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
View file

@ -750,14 +750,14 @@ public class BinaryDictionaryTests extends AndroidTestCase {
addBigramWords(binaryDictionary, word0, word1, bigramProbability);
}
assertEquals(new HashSet<String>(words).size(), Integer.parseInt(
binaryDictionary.getPropertyForTests(BinaryDictionary.UNIGRAM_COUNT_QUERY)));
binaryDictionary.getPropertyForTest(BinaryDictionary.UNIGRAM_COUNT_QUERY)));
assertEquals(new HashSet<Pair<String, String>>(bigrams).size(), Integer.parseInt(
binaryDictionary.getPropertyForTests(BinaryDictionary.BIGRAM_COUNT_QUERY)));
binaryDictionary.getPropertyForTest(BinaryDictionary.BIGRAM_COUNT_QUERY)));
binaryDictionary.flushWithGC();
assertEquals(new HashSet<String>(words).size(), Integer.parseInt(
binaryDictionary.getPropertyForTests(BinaryDictionary.UNIGRAM_COUNT_QUERY)));
binaryDictionary.getPropertyForTest(BinaryDictionary.UNIGRAM_COUNT_QUERY)));
assertEquals(new HashSet<Pair<String, String>>(bigrams).size(), Integer.parseInt(
binaryDictionary.getPropertyForTests(BinaryDictionary.BIGRAM_COUNT_QUERY)));
binaryDictionary.getPropertyForTest(BinaryDictionary.BIGRAM_COUNT_QUERY)));
binaryDictionary.close();
}