nyancrimew/LatinIME
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge "Split the header into a separate file."

Browse source
This commit is contained in:
Jean Chalard 2013-12-11 03:37:05 +00:00 committed by Android (Google) Code Review
commit a8ffdf0288
26 changed files with 155 additions and 121 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java
View file

@ -36,27 +36,25 @@ public abstract class AbstractDictDecoder implements DictDecoder {
private static final int ERROR_CANNOT_READ = 1;
private static final int ERROR_WRONG_FORMAT = 2;
protected FileHeader readHeader(final DictBuffer dictBuffer)
protected FileHeader readHeader(final DictBuffer headerBuffer)
throws IOException, UnsupportedFormatException {
if (dictBuffer == null) {
if (headerBuffer == null) {
openDictBuffer();
}
final int version = HeaderReader.readVersion(dictBuffer);
final int version = HeaderReader.readVersion(headerBuffer);
if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION
|| version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) {
throw new UnsupportedFormatException("Unsupported version : " + version);
}
// TODO: Remove this field.
final int optionsFlags = HeaderReader.readOptionFlags(dictBuffer);
final int headerSize = HeaderReader.readHeaderSize(dictBuffer);
final int optionsFlags = HeaderReader.readOptionFlags(headerBuffer);
final int headerSize = HeaderReader.readHeaderSize(headerBuffer);
if (headerSize < 0) {
throw new UnsupportedFormatException("header size can't be negative.");
}
final HashMap<String, String> attributes = HeaderReader.readAttributes(dictBuffer,
final HashMap<String, String> attributes = HeaderReader.readAttributes(headerBuffer,
headerSize);
final FileHeader header = new FileHeader(headerSize,

2
java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java
View file

@ -600,7 +600,7 @@ public final class BinaryDictDecoderUtils {
Map<Integer, PtNodeArray> reverseNodeArrayMapping = new TreeMap<Integer, PtNodeArray>();
Map<Integer, PtNode> reversePtNodeMapping = new TreeMap<Integer, PtNode>();
final PtNodeArray root = readNodeArray(dictDecoder, fileHeader.mHeaderSize,
final PtNodeArray root = readNodeArray(dictDecoder, fileHeader.mBodyOffset,
reverseNodeArrayMapping, reversePtNodeMapping, fileHeader.mFormatOptions);
FusionDictionary newDict = new FusionDictionary(root, fileHeader.mDictionaryOptions);

6
java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java
View file

@ -62,7 +62,7 @@ public final class BinaryDictIOUtils {
* Retrieves all node arrays without recursive call.
*/
private static void readUnigramsAndBigramsBinaryInner(final DictDecoder dictDecoder,
final int headerSize, final Map<Integer, String> words,
final int bodyOffset, final Map<Integer, String> words,
final Map<Integer, Integer> frequencies,
final Map<Integer, ArrayList<PendingAttribute>> bigrams,
final FormatOptions formatOptions) {
@ -71,7 +71,7 @@ public final class BinaryDictIOUtils {
Stack<Position> stack = new Stack<Position>();
int index = 0;
Position initPos = new Position(headerSize, 0);
Position initPos = new Position(bodyOffset, 0);
stack.push(initPos);
while (!stack.empty()) {
@ -154,7 +154,7 @@ public final class BinaryDictIOUtils {
UnsupportedFormatException {
// Read header
final FileHeader header = dictDecoder.readHeader();
readUnigramsAndBigramsBinaryInner(dictDecoder, header.mHeaderSize, words,
readUnigramsAndBigramsBinaryInner(dictDecoder, header.mBodyOffset, words,
frequencies, bigrams, header.mFormatOptions);
}

22
java/src/com/android/inputmethod/latin/makedict/FormatSpec.java
View file

@ -199,13 +199,19 @@ public final class FormatSpec {
*/
public static final int MAGIC_NUMBER = 0x9BC13AFE;
static final int MINIMUM_SUPPORTED_VERSION = 2;
static final int MAXIMUM_SUPPORTED_VERSION = 4;
static final int NOT_A_VERSION_NUMBER = -1;
static final int FIRST_VERSION_WITH_DYNAMIC_UPDATE = 3;
static final int FIRST_VERSION_WITH_TERMINAL_ID = 4;
// These MUST have the same values as the relevant constants in format_utils.h.
// From version 4 on, we use version * 100 + revision as a version number. That allows
// us to change the format during development while having testing devices remove
// older files with each upgrade, while still having a readable versioning scheme.
public static final int VERSION2 = 2;
public static final int VERSION3 = 3;
public static final int VERSION4 = 4;
public static final int VERSION4 = 400;
static final int MINIMUM_SUPPORTED_VERSION = VERSION2;
static final int MAXIMUM_SUPPORTED_VERSION = VERSION4;
// These options need to be the same numeric values as the one in the native reading code.
static final int GERMAN_UMLAUT_PROCESSING_FLAG = 0x1;
@ -263,8 +269,10 @@ public final class FormatSpec {
static final int PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE = 3;
static final int PTNODE_SHORTCUT_LIST_SIZE_SIZE = 2;
// These values are used only by version 4 or later.
public static final String TRIE_FILE_EXTENSION = ".trie";
// These values are used only by version 4 or later. They MUST match the definitions in
// ver4_dict_constants.cpp.
static final String TRIE_FILE_EXTENSION = ".trie";
public static final String HEADER_FILE_EXTENSION = ".header";
static final String FREQ_FILE_EXTENSION = ".freq";
static final String UNIGRAM_TIMESTAMP_FILE_EXTENSION = ".timestamp";
// tat = Terminal Address Table
@ -361,7 +369,7 @@ public final class FormatSpec {
* Class representing file header.
*/
public static final class FileHeader {
public final int mHeaderSize;
public final int mBodyOffset;
public final DictionaryOptions mDictionaryOptions;
public final FormatOptions mFormatOptions;
// Note that these are corresponding definitions in native code in latinime::HeaderPolicy
@ -377,9 +385,9 @@ public final class FormatSpec {
private static final String DICTIONARY_DESCRIPTION_ATTRIBUTE = "description";
public FileHeader(final int headerSize, final DictionaryOptions dictionaryOptions,
final FormatOptions formatOptions) {
mHeaderSize = headerSize;
mDictionaryOptions = dictionaryOptions;
mFormatOptions = formatOptions;
mBodyOffset = formatOptions.mVersion < VERSION4 ? headerSize : 0;
if (null == getLocaleString()) {
throw new RuntimeException("Cannot create a FileHeader without a locale");
}

20
java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java
View file

@ -45,10 +45,12 @@ public class Ver4DictDecoder extends AbstractDictDecoder {
protected static final int FILETYPE_TERMINAL_ADDRESS_TABLE = 3;
protected static final int FILETYPE_BIGRAM_FREQ = 4;
protected static final int FILETYPE_SHORTCUT = 5;
protected static final int FILETYPE_HEADER = 6;
protected final File mDictDirectory;
protected final DictionaryBufferFactory mBufferFactory;
protected DictBuffer mDictBuffer;
protected DictBuffer mHeaderBuffer;
protected DictBuffer mFrequencyBuffer;
protected DictBuffer mTerminalAddressTableBuffer;
private BigramContentReader mBigramReader;
@ -83,7 +85,7 @@ public class Ver4DictDecoder extends AbstractDictDecoder {
@UsedForTesting
/* package */ Ver4DictDecoder(final File dictDirectory, final int factoryFlag) {
mDictDirectory = dictDirectory;
mDictBuffer = mFrequencyBuffer = null;
mDictBuffer = mHeaderBuffer = mFrequencyBuffer = null;
if ((factoryFlag & MASK_DICTBUFFER) == USE_READONLY_BYTEBUFFER) {
mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory();
@ -100,13 +102,16 @@ public class Ver4DictDecoder extends AbstractDictDecoder {
/* package */ Ver4DictDecoder(final File dictDirectory, final DictionaryBufferFactory factory) {
mDictDirectory = dictDirectory;
mBufferFactory = factory;
mDictBuffer = mFrequencyBuffer = null;
mDictBuffer = mHeaderBuffer = mFrequencyBuffer = null;
}
protected File getFile(final int fileType) throws UnsupportedFormatException {
if (fileType == FILETYPE_TRIE) {
return new File(mDictDirectory,
mDictDirectory.getName() + FormatSpec.TRIE_FILE_EXTENSION);
} else if (fileType == FILETYPE_HEADER) {
return new File(mDictDirectory,
mDictDirectory.getName() + FormatSpec.HEADER_FILE_EXTENSION);
} else if (fileType == FILETYPE_FREQUENCY) {
return new File(mDictDirectory,
mDictDirectory.getName() + FormatSpec.FREQ_FILE_EXTENSION);
@ -132,6 +137,7 @@ public class Ver4DictDecoder extends AbstractDictDecoder {
if (!mDictDirectory.isDirectory()) {
throw new UnsupportedFormatException("Format 4 dictionary needs a directory");
}
mHeaderBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_HEADER));
mDictBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_TRIE));
mFrequencyBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_FREQUENCY));
mTerminalAddressTableBuffer = mBufferFactory.getDictionaryBuffer(
@ -149,6 +155,11 @@ public class Ver4DictDecoder extends AbstractDictDecoder {
return mDictBuffer != null;
}
@UsedForTesting
/* package */ DictBuffer getHeaderBuffer() {
return mHeaderBuffer;
}
@UsedForTesting
/* package */ DictBuffer getDictBuffer() {
return mDictBuffer;
@ -156,10 +167,11 @@ public class Ver4DictDecoder extends AbstractDictDecoder {
@Override
public FileHeader readHeader() throws IOException, UnsupportedFormatException {
if (mDictBuffer == null) {
if (mHeaderBuffer == null) {
openDictBuffer();
}
final FileHeader header = super.readHeader(mDictBuffer);
mHeaderBuffer.position(0);
final FileHeader header = super.readHeader(mHeaderBuffer);
final int version = header.mFormatOptions.mVersion;
if (version != FormatSpec.VERSION4) {
throw new UnsupportedFormatException("File header has a wrong version : " + version);

19
java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java
View file

@ -26,6 +26,7 @@ import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.FileUtils;
import java.io.File;
import java.io.FileNotFoundException;
@ -45,8 +46,8 @@ public class Ver4DictEncoder implements DictEncoder {
private final File mDictPlacedDir;
private byte[] mTrieBuf;
private int mTriePos;
private int mHeaderSize;
private OutputStream mTrieOutStream;
private OutputStream mHeaderOutStream;
private OutputStream mFreqOutStream;
private OutputStream mUnigramTimestampOutStream;
private OutputStream mTerminalAddressTableOutStream;
@ -185,16 +186,21 @@ public class Ver4DictEncoder implements DictEncoder {
mBaseFilename = header.getId() + "." + header.getVersion();
mDictDir = new File(mDictPlacedDir, mBaseFilename);
final File trieFile = new File(mDictDir, mBaseFilename + FormatSpec.TRIE_FILE_EXTENSION);
final File headerFile = new File(mDictDir,
mBaseFilename + FormatSpec.HEADER_FILE_EXTENSION);
final File freqFile = new File(mDictDir, mBaseFilename + FormatSpec.FREQ_FILE_EXTENSION);
final File timestampFile = new File(mDictDir,
mBaseFilename + FormatSpec.UNIGRAM_TIMESTAMP_FILE_EXTENSION);
final File terminalAddressTableFile = new File(mDictDir,
mBaseFilename + FormatSpec.TERMINAL_ADDRESS_TABLE_FILE_EXTENSION);
if (!mDictDir.isDirectory()) {
if (mDictDir.exists()) mDictDir.delete();
if (mDictDir.exists()) {
FileUtils.deleteRecursively(mDictDir);
}
mDictDir.mkdirs();
}
mTrieOutStream = new FileOutputStream(trieFile);
mHeaderOutStream = new FileOutputStream(headerFile);
mFreqOutStream = new FileOutputStream(freqFile);
mTerminalAddressTableOutStream = new FileOutputStream(terminalAddressTableFile);
if (formatOptions.mHasTimestamp) {
@ -207,6 +213,9 @@ public class Ver4DictEncoder implements DictEncoder {
if (mTrieOutStream != null) {
mTrieOutStream.close();
}
if (mHeaderOutStream != null) {
mHeaderOutStream.close();
}
if (mFreqOutStream != null) {
mFreqOutStream.close();
}
@ -218,6 +227,7 @@ public class Ver4DictEncoder implements DictEncoder {
}
} finally {
mTrieOutStream = null;
mHeaderOutStream = null;
mFreqOutStream = null;
mTerminalAddressTableOutStream = null;
}
@ -238,8 +248,7 @@ public class Ver4DictEncoder implements DictEncoder {
openStreams(formatOptions, dict.mOptions);
}
mHeaderSize = BinaryDictEncoderUtils.writeDictionaryHeader(mTrieOutStream, dict,
formatOptions);
BinaryDictEncoderUtils.writeDictionaryHeader(mHeaderOutStream, dict, formatOptions);
MakedictLog.i("Flattening the tree...");
ArrayList<PtNodeArray> flatNodes = BinaryDictEncoderUtils.flattenTree(dict.mRootNodeArray);
@ -423,7 +432,7 @@ public class Ver4DictEncoder implements DictEncoder {
ptNode.mFrequency, FormatSpec.FREQUENCY_AND_FLAGS_SIZE);
BinaryDictEncoderUtils.writeUIntToBuffer(terminalAddressTableBuf,
ptNode.mTerminalId * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE,
ptNode.mCachedAddressAfterUpdate + mHeaderSize,
ptNode.mCachedAddressAfterUpdate,
FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE);
}
}

3
java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
View file

@ -122,7 +122,8 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
@Override
protected String getFileNameExtentionToOpenDict() {
return "/" + FormatSpec.TRIE_FILE_EXTENSION;
// TODO: pass the directory name instead
return "/" + FormatSpec.HEADER_FILE_EXTENSION;
}
public void addMultipleDictionaryEntriesToDictionary(

2
native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp
View file

@ -47,7 +47,7 @@ namespace latinime {
case FormatUtils::VERSION_4: {
const int dictDirPathBufSize = strlen(path) + 1 /* terminator */;
char dictDirPath[dictDirPathBufSize];
if (!FileUtils::getFilePathWithoutSuffix(path, Ver4DictConstants::TRIE_FILE_EXTENSION,
if (!FileUtils::getFilePathWithoutSuffix(path, Ver4DictConstants::HEADER_FILE_EXTENSION,
dictDirPathBufSize, dictDirPath)) {
// Dictionary file name is not valid as a version 4 dictionary.
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0);

5
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h
View file

@ -60,9 +60,8 @@ class SingleDictContent : public DictContent {
}
bool flush(const char *const dictDirPath, const char *const contentFileName) const {
const BufferWithExtendableBuffer *bufferPtr = &mExpandableContentBuffer;
return DictFileWritingUtils::flushBuffersToFileInDir(dictDirPath, contentFileName,
&bufferPtr, 1 /* bufferCount */);
return DictFileWritingUtils::flushBufferToFileInDir(dictDirPath, contentFileName,
&mExpandableContentBuffer);
}
private:

15
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp
View file

@ -21,19 +21,16 @@ namespace latinime {
bool SparseTableDictContent::flush(const char *const dictDirPath,
const char *const lookupTableFileName, const char *const addressTableFileName,
const char *const contentFileName) const {
const BufferWithExtendableBuffer *lookupTableBufferPtr = &mExpandableLookupTableBuffer;
if (!DictFileWritingUtils::flushBuffersToFileInDir(dictDirPath, lookupTableFileName,
&lookupTableBufferPtr, 1 /* bufferCount */)) {
if (!DictFileWritingUtils::flushBufferToFileInDir(dictDirPath, lookupTableFileName,
&mExpandableLookupTableBuffer)){
return false;
}
const BufferWithExtendableBuffer *addressTableBufferPtr = &mExpandableAddressTableBuffer;
if (!DictFileWritingUtils::flushBuffersToFileInDir(dictDirPath, addressTableFileName,
&addressTableBufferPtr, 1 /* bufferCount */)) {
if (!DictFileWritingUtils::flushBufferToFileInDir(dictDirPath, addressTableFileName,
&mExpandableAddressTableBuffer)) {
return false;
}
const BufferWithExtendableBuffer *contentBufferPtr = &mExpandableContentBuffer;
if (!DictFileWritingUtils::flushBuffersToFileInDir(dictDirPath, contentFileName,
&contentBufferPtr, 1 /* bufferCount */)) {
if (!DictFileWritingUtils::flushBufferToFileInDir(dictDirPath, contentFileName,
&mExpandableContentBuffer)) {
return false;
}
return true;

18
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.cpp
View file

@ -28,7 +28,7 @@ int TerminalPositionLookupTable::getTerminalPtNodePosition(const int terminalId)
const int terminalPos = getBuffer()->readUint(
Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE, getEntryPos(terminalId));
return (terminalPos == Ver4DictConstants::NOT_A_TERMINAL_ADDRESS) ?
NOT_A_DICT_POS : terminalPos - mHeaderRegionSize;
NOT_A_DICT_POS : terminalPos;
}
bool TerminalPositionLookupTable::setTerminalPtNodePosition(
@ -45,18 +45,16 @@ bool TerminalPositionLookupTable::setTerminalPtNodePosition(
mSize++;
}
const int terminalPos = (terminalPtNodePos != NOT_A_DICT_POS) ?
terminalPtNodePos + mHeaderRegionSize : Ver4DictConstants::NOT_A_TERMINAL_ADDRESS;
terminalPtNodePos : Ver4DictConstants::NOT_A_TERMINAL_ADDRESS;
return getWritableBuffer()->writeUint(terminalPos,
Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE, getEntryPos(terminalId));
}
bool TerminalPositionLookupTable::flushToFile(const char *const dictDirPath,
const int newHeaderRegionSize) const {
const int headerRegionSizeDiff = newHeaderRegionSize - mHeaderRegionSize;
// If header region size has been changed or used buffer size is smaller than actual buffer
// size, regenerate lookup table and write the new table to file.
if (headerRegionSizeDiff != 0 || getEntryPos(mSize) < getBuffer()->getTailPosition()) {
TerminalPositionLookupTable lookupTableToWrite(newHeaderRegionSize);
bool TerminalPositionLookupTable::flushToFile(const char *const dictDirPath) const {
// If the used buffer size is smaller than the actual buffer size, regenerate the lookup
// table and write the new table to the file.
if (getEntryPos(mSize) < getBuffer()->getTailPosition()) {
TerminalPositionLookupTable lookupTableToWrite;
for (int i = 0; i < mSize; ++i) {
const int terminalPtNodePosition = getTerminalPtNodePosition(i);
if (!lookupTableToWrite.setTerminalPtNodePosition(i, terminalPtNodePosition)) {
@ -68,7 +66,7 @@ bool TerminalPositionLookupTable::flushToFile(const char *const dictDirPath,
return lookupTableToWrite.flush(dictDirPath,
Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION);
} else {
// We can simply use this lookup table because the header region size has not been
// We can simply use this lookup table because the buffer size has not been
// changed.
return flush(dictDirPath, Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION);
}

15
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h
View file

@ -28,19 +28,13 @@ class TerminalPositionLookupTable : public SingleDictContent {
public:
typedef hash_map_compat<int, int> TerminalIdMap;
// TODO: Quit using headerRegionSize.
TerminalPositionLookupTable(const char *const dictDirPath, const bool isUpdatable,
const int headerRegionSize)
TerminalPositionLookupTable(const char *const dictDirPath, const bool isUpdatable)
: SingleDictContent(dictDirPath,
Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION, isUpdatable),
mSize(getBuffer()->getTailPosition()
/ Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE),
mHeaderRegionSize(headerRegionSize) {}
/ Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE) {}
explicit TerminalPositionLookupTable(const int headerRegionSize)
: mSize(0), mHeaderRegionSize(headerRegionSize) {}
TerminalPositionLookupTable() : mSize(0), mHeaderRegionSize(0) {}
TerminalPositionLookupTable() : mSize(0) {}
int getTerminalPtNodePosition(const int terminalId) const;
@ -50,7 +44,7 @@ class TerminalPositionLookupTable : public SingleDictContent {
return mSize;
}
bool flushToFile(const char *const dictDirPath, const int newHeaderRegionSize) const;
bool flushToFile(const char *const dictDirPath) const;
bool runGCTerminalIds(TerminalIdMap *const terminalIdMap);
@ -62,7 +56,6 @@ class TerminalPositionLookupTable : public SingleDictContent {
}
int mSize;
const int mHeaderRegionSize;
};
} // namespace latinime
#endif // LATINIME_TERMINAL_POSITION_LOOKUP_TABLE_H

14
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp
View file

@ -45,16 +45,22 @@ bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath,
AKLOGE("Cannot create directory: %s. errno: %d.", tmpDirPath, errno);
return false;
}
// Write header file.
if (!DictFileWritingUtils::flushBufferToFileInDir(tmpDirPath,
Ver4DictConstants::HEADER_FILE_EXTENSION, headerBuffer)) {
AKLOGE("Dictionary header file %s/%s cannot be written.", tmpDirPath,
Ver4DictConstants::HEADER_FILE_EXTENSION);
return false;
}
// Write trie file.
const BufferWithExtendableBuffer *buffers[] = {headerBuffer, &mExpandableTrieBuffer};
if (!DictFileWritingUtils::flushBuffersToFileInDir(tmpDirPath,
Ver4DictConstants::TRIE_FILE_EXTENSION, buffers, 2 /* bufferCount */)) {
if (!DictFileWritingUtils::flushBufferToFileInDir(tmpDirPath,
Ver4DictConstants::TRIE_FILE_EXTENSION, &mExpandableTrieBuffer)) {
AKLOGE("Dictionary trie file %s/%s cannot be written.", tmpDirPath,
Ver4DictConstants::TRIE_FILE_EXTENSION);
return false;
}
// Write dictionary contents.
if (!mTerminalPositionLookupTable.flushToFile(tmpDirPath, headerBuffer->getTailPosition())) {
if (!mTerminalPositionLookupTable.flushToFile(tmpDirPath)) {
AKLOGE("Terminal position lookup table cannot be written. %s", tmpDirPath);
return false;
}

27
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h
View file

@ -34,9 +34,10 @@ class Ver4DictBuffers {
typedef ExclusiveOwnershipPointer<Ver4DictBuffers> Ver4DictBuffersPtr;
static AK_FORCE_INLINE Ver4DictBuffersPtr openVer4DictBuffers(const char *const dictDirPath,
const MmappedBuffer::MmappedBufferPtr &dictBuffer) {
const bool isUpdatable = dictBuffer.get() ? dictBuffer.get()->isUpdatable() : false;
return Ver4DictBuffersPtr(new Ver4DictBuffers(dictDirPath, dictBuffer, isUpdatable));
const MmappedBuffer::MmappedBufferPtr &headerBuffer) {
const bool isUpdatable = headerBuffer.get() ? headerBuffer.get()->isUpdatable() : false;
// TODO: take only dictDirPath, and open both header and trie files in the constructor below
return Ver4DictBuffersPtr(new Ver4DictBuffers(dictDirPath, headerBuffer, isUpdatable));
}
static AK_FORCE_INLINE Ver4DictBuffersPtr createVer4DictBuffers(
@ -121,16 +122,17 @@ class Ver4DictBuffers {
DISALLOW_COPY_AND_ASSIGN(Ver4DictBuffers);
AK_FORCE_INLINE Ver4DictBuffers(const char *const dictDirPath,
const MmappedBuffer::MmappedBufferPtr &dictBuffer, const bool isUpdatable)
: mDictBuffer(dictBuffer),
mHeaderPolicy(mDictBuffer.get()->getBuffer(), FormatUtils::VERSION_4),
mExpandableHeaderBuffer(dictBuffer.get()->getBuffer(), mHeaderPolicy.getSize(),
const MmappedBuffer::MmappedBufferPtr &headerBuffer, const bool isUpdatable)
: mHeaderBuffer(headerBuffer),
mDictBuffer(MmappedBuffer::openBuffer(dictDirPath,
Ver4DictConstants::TRIE_FILE_EXTENSION, isUpdatable)),
mHeaderPolicy(headerBuffer.get()->getBuffer(), FormatUtils::VERSION_4),
mExpandableHeaderBuffer(headerBuffer.get()->getBuffer(), mHeaderPolicy.getSize(),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mExpandableTrieBuffer(dictBuffer.get()->getBuffer() + mHeaderPolicy.getSize(),
dictBuffer.get()->getBufferSize() - mHeaderPolicy.getSize(),
mExpandableTrieBuffer(mDictBuffer.get()->getBuffer(),
mDictBuffer.get()->getBufferSize(),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
// TODO: Quit using header size.
mTerminalPositionLookupTable(dictDirPath, isUpdatable, mHeaderPolicy.getSize()),
mTerminalPositionLookupTable(dictDirPath, isUpdatable),
mProbabilityDictContent(dictDirPath, mHeaderPolicy.hasHistoricalInfoOfWords(),
isUpdatable),
mBigramDictContent(dictDirPath, mHeaderPolicy.hasHistoricalInfoOfWords(),
@ -139,7 +141,7 @@ class Ver4DictBuffers {
mIsUpdatable(isUpdatable) {}
AK_FORCE_INLINE Ver4DictBuffers(const HeaderPolicy *const headerPolicy)
: mDictBuffer(0), mHeaderPolicy(),
: mHeaderBuffer(0), mDictBuffer(0), mHeaderPolicy(),
mExpandableHeaderBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
mExpandableTrieBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
mTerminalPositionLookupTable(),
@ -147,6 +149,7 @@ class Ver4DictBuffers {
mBigramDictContent(headerPolicy->hasHistoricalInfoOfWords()), mShortcutDictContent(),
mIsUpdatable(true) {}
const MmappedBuffer::MmappedBufferPtr mHeaderBuffer;
const MmappedBuffer::MmappedBufferPtr mDictBuffer;
const HeaderPolicy mHeaderPolicy;
BufferWithExtendableBuffer mExpandableHeaderBuffer;

2
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp
View file

@ -18,7 +18,9 @@
namespace latinime {
// These values MUST match the definitions in FormatSpec.java.
const char *const Ver4DictConstants::TRIE_FILE_EXTENSION = ".trie";
const char *const Ver4DictConstants::HEADER_FILE_EXTENSION = ".header";
const char *const Ver4DictConstants::FREQ_FILE_EXTENSION = ".freq";
// tat = Terminal Address Table
const char *const Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION = ".tat";

1
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h
View file

@ -25,6 +25,7 @@ namespace latinime {
class Ver4DictConstants {
public:
static const char *const TRIE_FILE_EXTENSION;
static const char *const HEADER_FILE_EXTENSION;
static const char *const FREQ_FILE_EXTENSION;
static const char *const TERMINAL_ADDRESS_TABLE_FILE_EXTENSION;
static const char *const BIGRAM_FILE_EXTENSION;

33
native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
View file

@ -68,42 +68,43 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE =
char tmpFileName[tmpFileNameBufSize];
FileUtils::getFilePathWithSuffix(filePath, TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE,
tmpFileNameBufSize, tmpFileName);
const BufferWithExtendableBuffer *buffers[] = {dictHeader, dictBody};
if (!DictFileWritingUtils::flushBuffersToFile(tmpFileName, buffers, 2 /* bufferCount */)) {
if (!DictFileWritingUtils::flushBufferToFile(tmpFileName, dictHeader)) {
AKLOGE("Dictionary header cannot be written to %s.", tmpFileName);
return false;
}
if (!DictFileWritingUtils::flushBufferToFile(tmpFileName, dictBody)) {
AKLOGE("Dictionary structure cannot be written to %s.", tmpFileName);
return false;
}
if (rename(tmpFileName, filePath) != 0) {
AKLOGE("Dictionary file %s cannot be renamed to %s", tmpFileName, filePath);;
return false;
}
return true;
}
/* static */ bool DictFileWritingUtils::flushBuffersToFileInDir(const char *const dirPath,
const char *const fileName, const BufferWithExtendableBuffer **const buffers,
const int bufferCount) {
/* static */ bool DictFileWritingUtils::flushBufferToFileInDir(const char *const dirPath,
const char *const fileName, const BufferWithExtendableBuffer *const buffer) {
const int filePathBufSize = FileUtils::getFilePathBufSize(dirPath, fileName);
char filePath[filePathBufSize];
FileUtils::getFilePath(dirPath, fileName, filePathBufSize, filePath);
return flushBuffersToFile(filePath, buffers, bufferCount);
return flushBufferToFile(filePath, buffer);
}
/* static */ bool DictFileWritingUtils::flushBuffersToFile(const char *const filePath,
const BufferWithExtendableBuffer **const buffers, const int bufferCount) {
/* static */ bool DictFileWritingUtils::flushBufferToFile(const char *const filePath,
const BufferWithExtendableBuffer *const buffer) {
FILE *const file = fopen(filePath, "wb");
if (!file) {
AKLOGE("File %s cannot be opened.", filePath);
ASSERT(false);
return false;
}
for (int i = 0; i < bufferCount; ++i) {
if (!writeBufferToFile(file, buffers[i])) {
remove(filePath);
AKLOGE("Buffer cannot be written to the file %s. size: %d", filePath,
buffers[i]->getTailPosition());
ASSERT(false);
return false;
}
if (!writeBufferToFile(file, buffer)) {
remove(filePath);
AKLOGE("Buffer cannot be written to the file %s. size: %d", filePath,
buffer->getTailPosition());
ASSERT(false);
return false;
}
fclose(file);
return true;

8
native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h
View file

@ -37,8 +37,8 @@ class DictFileWritingUtils {
BufferWithExtendableBuffer *const dictHeader,
BufferWithExtendableBuffer *const dictBody);
static bool flushBuffersToFileInDir(const char *const dirPath, const char *const fileName,
const BufferWithExtendableBuffer **const buffers, const int bufferCount);
static bool flushBufferToFileInDir(const char *const dirPath, const char *const fileName,
const BufferWithExtendableBuffer *const buffer);
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(DictFileWritingUtils);
@ -46,8 +46,8 @@ class DictFileWritingUtils {
static bool createEmptyV4DictFile(const char *const filePath,
const HeaderReadWriteUtils::AttributeMap *const attributeMap);
static bool flushBuffersToFile(const char *const filePath,
const BufferWithExtendableBuffer **const buffers, const int bufferCount);
static bool flushBufferToFile(const char *const filePath,
const BufferWithExtendableBuffer *const buffer);
static bool writeBufferToFile(FILE *const file,
const BufferWithExtendableBuffer *const buffer);

3
native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h
View file

@ -29,9 +29,10 @@ namespace latinime {
class FormatUtils {
public:
enum FORMAT_VERSION {
// These MUST have the same values as the relevant constants in FormatSpec.java.
VERSION_2 = 2,
VERSION_3 = 3,
VERSION_4 = 4,
VERSION_4 = 400,
UNKNOWN_VERSION = -1
};

9
tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
View file

@ -22,6 +22,7 @@ import android.util.Pair;
import com.android.inputmethod.latin.makedict.CodePointUtils;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.utils.FileUtils;
import java.io.File;
import java.io.IOException;
@ -102,7 +103,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
private File createEmptyDictionaryAndGetFile(final String dictId,
final int formatVersion) throws IOException {
if (formatVersion == 4) {
if (formatVersion == FormatSpec.VERSION4) {
return createEmptyVer4DictionaryAndGetFile(dictId);
} else {
throw new IOException("Dictionary format version " + formatVersion
@ -112,7 +113,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
private File createEmptyVer4DictionaryAndGetFile(final String dictId) throws IOException {
final File file = File.createTempFile(dictId, TEST_DICT_FILE_EXTENSION,
getContext().getCacheDir());
file.delete();
FileUtils.deleteRecursively(file);
file.mkdir();
Map<String, String> attributeMap = new HashMap<String, String>();
attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE,
@ -123,10 +124,10 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
FormatSpec.VERSION4, attributeMap)) {
return new File(file, FormatSpec.TRIE_FILE_EXTENSION);
return new File(file, FormatSpec.HEADER_FILE_EXTENSION);
} else {
throw new IOException("Empty dictionary " + file.getAbsolutePath() + " "
+ FormatSpec.TRIE_FILE_EXTENSION + " cannot be created.");
+ FormatSpec.HEADER_FILE_EXTENSION + " cannot be created.");
}
}

4
tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
View file

@ -72,10 +72,10 @@ public class BinaryDictionaryTests extends AndroidTestCase {
FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
FormatSpec.VERSION4, attributeMap)) {
return new File(file, FormatSpec.TRIE_FILE_EXTENSION);
return new File(file, FormatSpec.HEADER_FILE_EXTENSION);
} else {
throw new IOException("Empty dictionary " + file.getAbsolutePath() + " "
+ FormatSpec.TRIE_FILE_EXTENSION + " cannot be created.");
+ FormatSpec.HEADER_FILE_EXTENSION + " cannot be created.");
}
}

2
tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java
View file

@ -523,7 +523,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
return null;
}
if (fileHeader == null) return null;
return BinaryDictDecoderUtils.getWordAtPosition(dictDecoder, fileHeader.mHeaderSize,
return BinaryDictDecoderUtils.getWordAtPosition(dictDecoder, fileHeader.mBodyOffset,
address, fileHeader.mFormatOptions).mWord;
}

2
tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java
View file

@ -216,7 +216,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file);
final FileHeader fileHeader = dictDecoder.readHeader();
assertEquals(word,
BinaryDictDecoderUtils.getWordAtPosition(dictDecoder, fileHeader.mHeaderSize,
BinaryDictDecoderUtils.getWordAtPosition(dictDecoder, fileHeader.mBodyOffset,
position, fileHeader.mFormatOptions).mWord);
} catch (IOException e) {
Log.e(TAG, "Raised an IOException while looking up a word", e);

23
tests/src/com/android/inputmethod/latin/makedict/BinaryDictUtils.java
View file

@ -29,17 +29,18 @@ public class BinaryDictUtils {
public static final String TEST_DICT_FILE_EXTENSION = ".testDict";
public static final FormatSpec.FormatOptions VERSION2 = new FormatSpec.FormatOptions(2);
public static final FormatSpec.FormatOptions VERSION2 =
new FormatSpec.FormatOptions(FormatSpec.VERSION2);
public static final FormatSpec.FormatOptions VERSION3_WITHOUT_DYNAMIC_UPDATE =
new FormatSpec.FormatOptions(3, false /* supportsDynamicUpdate */);
new FormatSpec.FormatOptions(FormatSpec.VERSION3, false /* supportsDynamicUpdate */);
public static final FormatSpec.FormatOptions VERSION3_WITH_DYNAMIC_UPDATE =
new FormatSpec.FormatOptions(3, true /* supportsDynamicUpdate */);
new FormatSpec.FormatOptions(FormatSpec.VERSION3, true /* supportsDynamicUpdate */);
public static final FormatSpec.FormatOptions VERSION4_WITHOUT_DYNAMIC_UPDATE =
new FormatSpec.FormatOptions(4, false /* supportsDynamicUpdate */);
new FormatSpec.FormatOptions(FormatSpec.VERSION4, false /* supportsDynamicUpdate */);
public static final FormatSpec.FormatOptions VERSION4_WITH_DYNAMIC_UPDATE =
new FormatSpec.FormatOptions(4, true /* supportsDynamicUpdate */);
new FormatSpec.FormatOptions(FormatSpec.VERSION4, true /* supportsDynamicUpdate */);
public static final FormatSpec.FormatOptions VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP =
new FormatSpec.FormatOptions(4, true /* supportsDynamicUpdate */,
new FormatSpec.FormatOptions(FormatSpec.VERSION4, true /* supportsDynamicUpdate */,
true /* hasTimestamp */);
public static DictionaryOptions makeDictionaryOptions(final String id, final String version) {
@ -53,9 +54,10 @@ public class BinaryDictUtils {
public static File getDictFile(final String name, final String version,
final FormatOptions formatOptions, final File directory) {
if (formatOptions.mVersion == 2 || formatOptions.mVersion == 3) {
if (formatOptions.mVersion == FormatSpec.VERSION2
|| formatOptions.mVersion == FormatSpec.VERSION3) {
return new File(directory, name + "." + version + TEST_DICT_FILE_EXTENSION);
} else if (formatOptions.mVersion == 4) {
} else if (formatOptions.mVersion == FormatSpec.VERSION4) {
return new File(directory, name + "." + version);
} else {
throw new RuntimeException("the format option has a wrong version : "
@ -67,7 +69,8 @@ public class BinaryDictUtils {
final File cacheDir) {
if (formatOptions.mVersion == FormatSpec.VERSION4) {
return new Ver4DictEncoder(cacheDir);
} else if (formatOptions.mVersion == 3 || formatOptions.mVersion == 2) {
} else if (formatOptions.mVersion == FormatSpec.VERSION3
|| formatOptions.mVersion == FormatSpec.VERSION2) {
return new Ver3DictEncoder(file);
} else {
throw new RuntimeException("The format option has a wrong version : "
@ -79,7 +82,7 @@ public class BinaryDictUtils {
throws UnsupportedFormatException {
if (formatOptions.mVersion == FormatSpec.VERSION4) {
return new Ver4DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER);
} else if (formatOptions.mVersion == 3) {
} else if (formatOptions.mVersion == FormatSpec.VERSION3) {
return new Ver3DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER);
} else {
throw new UnsupportedFormatException("The format option has a wrong version : "

1
tools/dicttool/Android.mk
View file

@ -30,6 +30,7 @@ MAKEDICT_CORE_SOURCE_DIRECTORY := $(LATINIME_CORE_SOURCE_DIRECTORY)/makedict
USED_TARGETTED_UTILS := \
$(LATINIME_CORE_SOURCE_DIRECTORY)/utils/ByteArrayDictBuffer.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/utils/CollectionUtils.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/utils/FileUtils.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/utils/JniUtils.java
DICTTOOL_ONDEVICE_TESTS_DIRECTORY := \

6
tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java
View file

@ -159,9 +159,9 @@ public class DictionaryMaker {
if (OPTION_VERSION_2.equals(arg)) {
// Do nothing, this is the default
} else if (OPTION_VERSION_3.equals(arg)) {
outputBinaryFormatVersion = 3;
outputBinaryFormatVersion = FormatSpec.VERSION3;
} else if (OPTION_VERSION_4.equals(arg)) {
outputBinaryFormatVersion = 4;
outputBinaryFormatVersion = FormatSpec.VERSION4;
} else if (OPTION_HELP.equals(arg)) {
displayHelp();
} else {
@ -358,7 +358,7 @@ public class DictionaryMaker {
final File outputFile = new File(outputFilename);
final FormatSpec.FormatOptions formatOptions = new FormatSpec.FormatOptions(version);
final DictEncoder dictEncoder;
if (version == 4) {
if (version == FormatSpec.VERSION4) {
dictEncoder = new Ver4DictEncoder(outputFile);
} else {
dictEncoder = new Ver3DictEncoder(outputFile);