nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge "Add dictionary header writing methods."

main
Keisuke Kuroyanagi 2013-09-18 09:17:54 +00:00 committed by Android (Google) Code Review
parent 957a2337ee 484fa7b59c
commit 561a83118c
12 changed files with 181 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp
View File

@ -250,7 +250,7 @@ void DynamicPatriciaTriePolicy::flush(const char *const filePath) {
}
DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer,
&mBigramListPolicy, &mShortcutListPolicy);
writingHelper.writeToDictFile(filePath, mBuffer->getBuffer(), mHeaderPolicy.getSize());
writingHelper.writeToDictFile(filePath, &mHeaderPolicy);
}
void DynamicPatriciaTriePolicy::flushWithGC(const char *const filePath) {

2
native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h
View File

@ -33,7 +33,7 @@ class DicNodeVector;
class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
public:
DynamicPatriciaTriePolicy(const MmappedBuffer *const buffer)
: mBuffer(buffer), mHeaderPolicy(mBuffer->getBuffer()),
: mBuffer(buffer), mHeaderPolicy(mBuffer->getBuffer(), buffer->getBufferSize()),
mBufferWithExtendableBuffer(mBuffer->getBuffer() + mHeaderPolicy.getSize(),
mBuffer->getBufferSize() - mHeaderPolicy.getSize()),
mShortcutListPolicy(&mBufferWithExtendableBuffer),

10
native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp
View File

@ -24,6 +24,7 @@
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h"
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.h"
#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h"
@ -137,7 +138,11 @@ bool DynamicPatriciaTrieWritingHelper::removeBigramWords(const int word0Pos, con
}
void DynamicPatriciaTrieWritingHelper::writeToDictFile(const char *const fileName,
const uint8_t *const headerBuf, const int headerSize) {
const HeaderPolicy *const headerPolicy) {
BufferWithExtendableBuffer headerBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */);
if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, false /* updatesLastUpdatedTime */)) {
return;
}
const int tmpFileNameBufSize = strlen(fileName)
+ strlen(TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE) + 1;
char tmpFileName[tmpFileNameBufSize];
@ -148,7 +153,8 @@ void DynamicPatriciaTrieWritingHelper::writeToDictFile(const char *const fileNam
return;
}
// Write header.
if (fwrite(headerBuf, headerSize, 1, file) < 1) {
if (fwrite(headerBuffer.getBuffer(true /* usesAdditionalBuffer */),
headerBuffer.getTailPosition(), 1, file) < 1) {
fclose(file);
remove(tmpFileName);
return;

6
native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h
View File

@ -28,6 +28,7 @@ class DynamicBigramListPolicy;
class DynamicPatriciaTrieNodeReader;
class DynamicPatriciaTrieReadingHelper;
class DynamicShortcutListPolicy;
class HeaderPolicy;
class DynamicPatriciaTrieWritingHelper {
public:
@ -48,8 +49,9 @@ class DynamicPatriciaTrieWritingHelper {
// Remove a bigram relation from word0Pos to word1Pos.
bool removeBigramWords(const int word0Pos, const int word1Pos);
void writeToDictFile(const char *const fileName, const uint8_t *const headerBuf,
const int headerSize);
void writeToDictFile(const char *const fileName, const HeaderPolicy *const headerPolicy);
void writeToDictFileWithGC(const char *const fileName, const HeaderPolicy *const headerPolicy);
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieWritingHelper);

60
native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp
View File

@ -17,6 +17,8 @@
#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include <cstddef>
#include <cstdio>
#include <ctime>
namespace latinime {
@ -36,7 +38,7 @@ void HeaderPolicy::readHeaderValueOrQuestionMark(const char *const key, int *out
}
std::vector<int> keyCodePointVector;
insertCharactersIntoVector(key, &keyCodePointVector);
HeaderReadingUtils::AttributeMap::const_iterator it = mAttributeMap.find(keyCodePointVector);
HeaderReadWriteUtils::AttributeMap::const_iterator it = mAttributeMap.find(keyCodePointVector);
if (it == mAttributeMap.end()) {
// The key was not found.
outValue[0] = '?';
@ -85,7 +87,7 @@ int HeaderPolicy::readLastUpdatedTime() const {
bool HeaderPolicy::getAttributeValueAsInt(const char *const key, int *const outValue) const {
std::vector<int> keyVector;
insertCharactersIntoVector(key, &keyVector);
HeaderReadingUtils::AttributeMap::const_iterator it = mAttributeMap.find(keyVector);
HeaderReadWriteUtils::AttributeMap::const_iterator it = mAttributeMap.find(keyVector);
if (it == mAttributeMap.end()) {
// The key was not found.
return false;
@ -94,10 +96,56 @@ bool HeaderPolicy::getAttributeValueAsInt(const char *const key, int *const outV
return true;
}
/* static */ HeaderReadingUtils::AttributeMap HeaderPolicy::createAttributeMapAndReadAllAttributes(
const uint8_t *const dictBuf) {
HeaderReadingUtils::AttributeMap attributeMap;
HeaderReadingUtils::fetchAllHeaderAttributes(dictBuf, &attributeMap);
bool HeaderPolicy::writeHeaderToBuffer(BufferWithExtendableBuffer *const bufferToWrite,
const bool updatesLastUpdatedTime) const {
int writingPos = 0;
if (!HeaderReadWriteUtils::writeDictionaryVersion(bufferToWrite, mDictFormatVersion,
&writingPos)) {
return false;
}
if (!HeaderReadWriteUtils::writeDictionaryFlags(bufferToWrite, mDictionaryFlags,
&writingPos)) {
return false;
}
// Temporarily writes a dummy header size.
int headerSizeFieldPos = writingPos;
if (!HeaderReadWriteUtils::writeDictionaryHeaderSize(bufferToWrite, 0 /* size */,
&writingPos)) {
return false;
}
if (updatesLastUpdatedTime) {
// Set current time as a last updated time.
HeaderReadWriteUtils::AttributeMap attributeMapTowrite(mAttributeMap);
std::vector<int> updatedTimekey;
insertCharactersIntoVector(LAST_UPDATED_TIME_KEY, &updatedTimekey);
const time_t currentTime = time(NULL);
std::vector<int> updatedTimeValue;
char charBuf[LARGEST_INT_DIGIT_COUNT + 1];
snprintf(charBuf, LARGEST_INT_DIGIT_COUNT + 1, "%ld", currentTime);
insertCharactersIntoVector(charBuf, &updatedTimeValue);
attributeMapTowrite[updatedTimekey] = updatedTimeValue;
if (!HeaderReadWriteUtils::writeHeaderAttributes(bufferToWrite, &attributeMapTowrite,
&writingPos)) {
return false;
}
} else {
if (!HeaderReadWriteUtils::writeHeaderAttributes(bufferToWrite, &mAttributeMap,
&writingPos)) {
return false;
}
}
// Writes an actual header size.
if (!HeaderReadWriteUtils::writeDictionaryHeaderSize(bufferToWrite, writingPos,
&headerSizeFieldPos)) {
return false;
}
return true;
}
/* static */ HeaderReadWriteUtils::AttributeMap
HeaderPolicy::createAttributeMapAndReadAllAttributes(const uint8_t *const dictBuf) {
HeaderReadWriteUtils::AttributeMap attributeMap;
HeaderReadWriteUtils::fetchAllHeaderAttributes(dictBuf, &attributeMap);
return attributeMap;
}

26
native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
View File

@ -23,14 +23,17 @@
#include "defines.h"
#include "suggest/core/policy/dictionary_header_structure_policy.h"
#include "suggest/policyimpl/dictionary/header/header_reading_utils.h"
#include "suggest/policyimpl/dictionary/utils/format_utils.h"
namespace latinime {
class HeaderPolicy : public DictionaryHeaderStructurePolicy {
public:
explicit HeaderPolicy(const uint8_t *const dictBuf)
: mDictBuf(dictBuf), mDictionaryFlags(HeaderReadingUtils::getFlags(dictBuf)),
mSize(HeaderReadingUtils::getHeaderSize(dictBuf)),
explicit HeaderPolicy(const uint8_t *const dictBuf, const int dictSize)
: mDictBuf(dictBuf),
mDictFormatVersion(FormatUtils::detectFormatVersion(dictBuf, dictSize)),
mDictionaryFlags(HeaderReadWriteUtils::getFlags(dictBuf)),
mSize(HeaderReadWriteUtils::getHeaderSize(dictBuf)),
mAttributeMap(createAttributeMapAndReadAllAttributes(mDictBuf)),
mMultiWordCostMultiplier(readMultipleWordCostMultiplier()),
mUsesForgettingCurve(readUsesForgettingCurveFlag()),
@ -43,16 +46,15 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
}
AK_FORCE_INLINE bool supportsDynamicUpdate() const {
return HeaderReadingUtils::supportsDynamicUpdate(mDictionaryFlags);
return HeaderReadWriteUtils::supportsDynamicUpdate(mDictionaryFlags);
}
AK_FORCE_INLINE bool requiresGermanUmlautProcessing() const {
return HeaderReadingUtils::requiresGermanUmlautProcessing(mDictionaryFlags);
return HeaderReadWriteUtils::requiresGermanUmlautProcessing(mDictionaryFlags);
}
AK_FORCE_INLINE bool requiresFrenchLigatureProcessing() const {
return HeaderReadingUtils::requiresFrenchLigatureProcessing(
mDictionaryFlags);
return HeaderReadWriteUtils::requiresFrenchLigatureProcessing(mDictionaryFlags);
}
AK_FORCE_INLINE float getMultiWordCostMultiplier() const {
@ -70,6 +72,9 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
void readHeaderValueOrQuestionMark(const char *const key,
int *outValue, int outValueSize) const;
bool writeHeaderToBuffer(BufferWithExtendableBuffer *const bufferToWrite,
const bool updatesLastUpdatedTime) const;
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(HeaderPolicy);
@ -80,9 +85,10 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
static const float MULTIPLE_WORD_COST_MULTIPLIER_SCALE;
const uint8_t *const mDictBuf;
const HeaderReadingUtils::DictionaryFlags mDictionaryFlags;
const FormatUtils::FORMAT_VERSION mDictFormatVersion;
const HeaderReadWriteUtils::DictionaryFlags mDictionaryFlags;
const int mSize;
HeaderReadingUtils::AttributeMap mAttributeMap;
HeaderReadWriteUtils::AttributeMap mAttributeMap;
const float mMultiWordCostMultiplier;
const bool mUsesForgettingCurve;
const int mLastUpdatedTime;
@ -95,7 +101,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
bool getAttributeValueAsInt(const char *const key, int *const outValue) const;
static HeaderReadingUtils::AttributeMap createAttributeMapAndReadAllAttributes(
static HeaderReadWriteUtils::AttributeMap createAttributeMapAndReadAllAttributes(
const uint8_t *const dictBuf);
static int parseIntAttributeValue(const std::vector<int> *const attributeValue);

84
native/jni/src/suggest/policyimpl/dictionary/header/header_reading_utils.cpp
View File

@ -19,43 +19,44 @@
#include <vector>
#include "defines.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h"
namespace latinime {
const int HeaderReadingUtils::MAX_ATTRIBUTE_KEY_LENGTH = 256;
const int HeaderReadingUtils::MAX_ATTRIBUTE_VALUE_LENGTH = 256;
const int HeaderReadWriteUtils::MAX_ATTRIBUTE_KEY_LENGTH = 256;
const int HeaderReadWriteUtils::MAX_ATTRIBUTE_VALUE_LENGTH = 256;
const int HeaderReadingUtils::HEADER_MAGIC_NUMBER_SIZE = 4;
const int HeaderReadingUtils::HEADER_DICTIONARY_VERSION_SIZE = 2;
const int HeaderReadingUtils::HEADER_FLAG_SIZE = 2;
const int HeaderReadingUtils::HEADER_SIZE_FIELD_SIZE = 4;
const int HeaderReadWriteUtils::HEADER_MAGIC_NUMBER_SIZE = 4;
const int HeaderReadWriteUtils::HEADER_DICTIONARY_VERSION_SIZE = 2;
const int HeaderReadWriteUtils::HEADER_FLAG_SIZE = 2;
const int HeaderReadWriteUtils::HEADER_SIZE_FIELD_SIZE = 4;
const HeaderReadingUtils::DictionaryFlags HeaderReadingUtils::NO_FLAGS = 0;
const HeaderReadWriteUtils::DictionaryFlags HeaderReadWriteUtils::NO_FLAGS = 0;
// Flags for special processing
// Those *must* match the flags in makedict (FormatSpec#*_PROCESSING_FLAG) or
// something very bad (like, the apocalypse) will happen. Please update both at the same time.
const HeaderReadingUtils::DictionaryFlags
HeaderReadingUtils::GERMAN_UMLAUT_PROCESSING_FLAG = 0x1;
const HeaderReadingUtils::DictionaryFlags
HeaderReadingUtils::SUPPORTS_DYNAMIC_UPDATE_FLAG = 0x2;
const HeaderReadingUtils::DictionaryFlags
HeaderReadingUtils::FRENCH_LIGATURE_PROCESSING_FLAG = 0x4;
const HeaderReadWriteUtils::DictionaryFlags
HeaderReadWriteUtils::GERMAN_UMLAUT_PROCESSING_FLAG = 0x1;
const HeaderReadWriteUtils::DictionaryFlags
HeaderReadWriteUtils::SUPPORTS_DYNAMIC_UPDATE_FLAG = 0x2;
const HeaderReadWriteUtils::DictionaryFlags
HeaderReadWriteUtils::FRENCH_LIGATURE_PROCESSING_FLAG = 0x4;
/* static */ int HeaderReadingUtils::getHeaderSize(const uint8_t *const dictBuf) {
/* static */ int HeaderReadWriteUtils::getHeaderSize(const uint8_t *const dictBuf) {
// See the format of the header in the comment in
// BinaryDictionaryFormatUtils::detectFormatVersion()
return ByteArrayUtils::readUint32(dictBuf, HEADER_MAGIC_NUMBER_SIZE
+ HEADER_DICTIONARY_VERSION_SIZE + HEADER_FLAG_SIZE);
}
/* static */ HeaderReadingUtils::DictionaryFlags
HeaderReadingUtils::getFlags(const uint8_t *const dictBuf) {
/* static */ HeaderReadWriteUtils::DictionaryFlags
HeaderReadWriteUtils::getFlags(const uint8_t *const dictBuf) {
return ByteArrayUtils::readUint16(dictBuf,
HEADER_MAGIC_NUMBER_SIZE + HEADER_DICTIONARY_VERSION_SIZE);
}
/* static */ void HeaderReadingUtils::fetchAllHeaderAttributes(const uint8_t *const dictBuf,
/* static */ void HeaderReadWriteUtils::fetchAllHeaderAttributes(const uint8_t *const dictBuf,
AttributeMap *const headerAttributes) {
const int headerSize = getHeaderSize(dictBuf);
int pos = getHeaderOptionsPosition();
@ -78,4 +79,53 @@ const HeaderReadingUtils::DictionaryFlags
}
}
/* static */ bool HeaderReadWriteUtils::writeDictionaryVersion(
BufferWithExtendableBuffer *const buffer, const FormatUtils::FORMAT_VERSION version,
int *const writingPos) {
if (!buffer->writeUintAndAdvancePosition(FormatUtils::MAGIC_NUMBER, HEADER_MAGIC_NUMBER_SIZE,
writingPos)) {
return false;
}
switch (version) {
case FormatUtils::VERSION_2:
// Version 2 dictionary writing is not supported.
return false;
case FormatUtils::VERSION_3:
return buffer->writeUintAndAdvancePosition(3 /* data */,
HEADER_DICTIONARY_VERSION_SIZE, writingPos);
default:
return false;
}
}
/* static */ bool HeaderReadWriteUtils::writeDictionaryFlags(
BufferWithExtendableBuffer *const buffer, const DictionaryFlags flags,
int *const writingPos) {
return buffer->writeUintAndAdvancePosition(flags, HEADER_FLAG_SIZE, writingPos);
}
/* static */ bool HeaderReadWriteUtils::writeDictionaryHeaderSize(
BufferWithExtendableBuffer *const buffer, const int size, int *const writingPos) {
return buffer->writeUintAndAdvancePosition(size, HEADER_SIZE_FIELD_SIZE, writingPos);
}
/* static */ bool HeaderReadWriteUtils::writeHeaderAttributes(
BufferWithExtendableBuffer *const buffer, const AttributeMap *const headerAttributes,
int *const writingPos) {
for (AttributeMap::const_iterator it = headerAttributes->begin();
it != headerAttributes->end(); ++it) {
// Write a key.
if (!buffer->writeCodePointsAndAdvancePosition(&(it->first.at(0)), it->first.size(),
true /* writesTerminator */, writingPos)) {
return false;
}
// Write a value.
if (!buffer->writeCodePointsAndAdvancePosition(&(it->second.at(0)), it->second.size(),
true /* writesTerminator */, writingPos)) {
return false;
}
}
return true;
}
} // namespace latinime

20
native/jni/src/suggest/policyimpl/dictionary/header/header_reading_utils.h
View File

@ -22,10 +22,14 @@
#include <vector>
#include "defines.h"
#include "suggest/policyimpl/dictionary/utils/format_utils.h"
namespace latinime {
class HeaderReadingUtils {
class BufferWithExtendableBuffer;
// TODO: Change this file name to header_read_write_utils.h.
class HeaderReadWriteUtils {
public:
typedef uint16_t DictionaryFlags;
typedef std::map<std::vector<int>, std::vector<int> > AttributeMap;
@ -54,8 +58,20 @@ class HeaderReadingUtils {
static void fetchAllHeaderAttributes(const uint8_t *const dictBuf,
AttributeMap *const headerAttributes);
static bool writeDictionaryVersion(BufferWithExtendableBuffer *const buffer,
const FormatUtils::FORMAT_VERSION version, int *const writingPos);
static bool writeDictionaryFlags(BufferWithExtendableBuffer *const buffer,
const DictionaryFlags flags, int *const writingPos);
static bool writeDictionaryHeaderSize(BufferWithExtendableBuffer *const buffer,
const int size, int *const writingPos);
static bool writeHeaderAttributes(BufferWithExtendableBuffer *const buffer,
const AttributeMap *const headerAttributes, int *const writingPos);
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(HeaderReadingUtils);
DISALLOW_IMPLICIT_CONSTRUCTORS(HeaderReadWriteUtils);
static const int MAX_ATTRIBUTE_KEY_LENGTH;
static const int MAX_ATTRIBUTE_VALUE_LENGTH;

2
native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h
View File

@ -34,7 +34,7 @@ class DicNodeVector;
class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
public:
PatriciaTriePolicy(const MmappedBuffer *const buffer)
: mBuffer(buffer), mHeaderPolicy(mBuffer->getBuffer()),
: mBuffer(buffer), mHeaderPolicy(mBuffer->getBuffer(), buffer->getBufferSize()),
mDictRoot(mBuffer->getBuffer() + mHeaderPolicy.getSize()),
mBigramListPolicy(mDictRoot), mShortcutListPolicy(mDictRoot) {}

26
native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp
View File

@ -20,20 +20,10 @@
namespace latinime {
/**
* Dictionary size
*/
// Any file smaller than this is not a dictionary.
const int FormatUtils::DICTIONARY_MINIMUM_SIZE = 4;
const uint32_t FormatUtils::MAGIC_NUMBER = 0x9BC13AFE;
/**
* Format versions
*/
// 32 bit magic number is stored at the beginning of the dictionary header to reject unsupported
// or obsolete dictionary formats.
const uint32_t FormatUtils::HEADER_VERSION_2_MAGIC_NUMBER = 0x9BC13AFE;
// Magic number (4 bytes), version (2 bytes), options (2 bytes), header size (4 bytes) = 12
const int FormatUtils::HEADER_VERSION_2_MINIMUM_SIZE = 12;
// Magic number (4 bytes), version (2 bytes), flags (2 bytes), header size (4 bytes) = 12
const int FormatUtils::DICTIONARY_MINIMUM_SIZE = 12;
/* static */ FormatUtils::FORMAT_VERSION FormatUtils::detectFormatVersion(
const uint8_t *const dict, const int dictSize) {
@ -45,16 +35,10 @@ const int FormatUtils::HEADER_VERSION_2_MINIMUM_SIZE = 12;
}
const uint32_t magicNumber = ByteArrayUtils::readUint32(dict, 0);
switch (magicNumber) {
case HEADER_VERSION_2_MAGIC_NUMBER:
// Version 2 header are at least 12 bytes long.
// If this header has the version 2 magic number but is less than 12 bytes long,
// then it's an unknown format and we need to avoid confidently reading the next bytes.
if (dictSize < HEADER_VERSION_2_MINIMUM_SIZE) {
return UNKNOWN_VERSION;
}
case MAGIC_NUMBER:
// Version 2 header is as follows:
// Magic number (4 bytes) 0x9B 0xC1 0x3A 0xFE
// Version number (2 bytes)
// Dictionary format version number (2 bytes)
// Options (2 bytes)
// Header size (4 bytes) : integer, big endian
if (ByteArrayUtils::readUint16(dict, 4) == 2) {

6
native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h
View File

@ -34,14 +34,16 @@ class FormatUtils {
UNKNOWN_VERSION
};
// 32 bit magic number is stored at the beginning of the dictionary header to reject
// unsupported or obsolete dictionary formats.
static const uint32_t MAGIC_NUMBER;
static FORMAT_VERSION detectFormatVersion(const uint8_t *const dict, const int dictSize);
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(FormatUtils);
static const int DICTIONARY_MINIMUM_SIZE;
static const uint32_t HEADER_VERSION_2_MAGIC_NUMBER;
static const int HEADER_VERSION_2_MINIMUM_SIZE;
};
} // namespace latinime
#endif /* LATINIME_FORMAT_UTILS_H */

4
tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
View File

@ -358,8 +358,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
assertEquals(-1, binaryDictionary.getFrequency("aaa"));
assertEquals(-1, binaryDictionary.getFrequency("abcd"));
assertEquals(Dictionary.NOT_A_PROBABILITY, binaryDictionary.getFrequency("aaa"));
assertEquals(Dictionary.NOT_A_PROBABILITY, binaryDictionary.getFrequency("abcd"));
binaryDictionary.addUnigramWord("aaa", probability);
binaryDictionary.addUnigramWord("abcd", probability);