nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Have dicttool use the native library to generate v4 dicts. 

Yay !

Change-Id: Iea8ced9e81031b9ab7eff05ad9ef7215be248de9
main
Jean Chalard 2013-12-12 15:08:10 +09:00
parent af0c222a5e
commit a245d15da5
38 changed files with 580 additions and 591 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java
View File

@ -21,7 +21,7 @@ import android.util.Log;
import com.android.inputmethod.latin.makedict.DictEncoder;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
import com.android.inputmethod.latin.makedict.Ver2DictEncoder;
import java.io.File;
import java.io.IOException;
@ -64,7 +64,7 @@ abstract public class AbstractDictionaryWriter {
final String tempFilePath = file.getAbsolutePath() + ".temp";
final File tempFile = new File(tempFilePath);
try {
final DictEncoder dictEncoder = new Ver3DictEncoder(tempFile);
final DictEncoder dictEncoder = new Ver2DictEncoder(tempFile);
writeDictionary(dictEncoder, attributeMap);
tempFile.renameTo(file);
} catch (IOException e) {

3
java/src/com/android/inputmethod/latin/BinaryDictionary.java
View File

@ -367,6 +367,7 @@ public final class BinaryDictionary extends Dictionary {
public static class LanguageModelParam {
public final int[] mWord0;
public final int[] mWord1;
// TODO: this needs to be a list of shortcuts
public final int[] mShortcutTarget;
public final int mUnigramProbability;
public final int mBigramProbability;
@ -375,7 +376,7 @@ public final class BinaryDictionary extends Dictionary {
public final boolean mIsBlacklisted;
public final int mTimestamp;
// Constructor for unigram.
// Constructor for unigram. TODO: support shortcuts
public LanguageModelParam(final String word, final int unigramProbability,
final int timestamp) {
mWord0 = null;

14
java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java
View File

@ -55,7 +55,7 @@ public final class DynamicBinaryDictIOUtils {
* @param newParentAddress the absolute address of the parent.
* @param formatOptions file format options.
*/
private static void updateParentAddress(final Ver3DictUpdater dictUpdater,
private static void updateParentAddress(final Ver2DictUpdater dictUpdater,
final int ptNodeOriginAddress, final int newParentAddress,
final FormatOptions formatOptions) {
final DictBuffer dictBuffer = dictUpdater.getDictBuffer();
@ -88,7 +88,7 @@ public final class DynamicBinaryDictIOUtils {
* @param newParentAddress the address to be written.
* @param formatOptions file format options.
*/
private static void updateParentAddresses(final Ver3DictUpdater dictUpdater,
private static void updateParentAddresses(final Ver2DictUpdater dictUpdater,
final int ptNodeOriginAddress, final int newParentAddress,
final FormatOptions formatOptions) {
final int originalPosition = dictUpdater.getPosition();
@ -114,7 +114,7 @@ public final class DynamicBinaryDictIOUtils {
* @param newChildrenAddress the absolute address of the child.
* @param formatOptions file format options.
*/
private static void updateChildrenAddress(final Ver3DictUpdater dictUpdater,
private static void updateChildrenAddress(final Ver2DictUpdater dictUpdater,
final int ptNodeOriginAddress, final int newChildrenAddress,
final FormatOptions formatOptions) {
final DictBuffer dictBuffer = dictUpdater.getDictBuffer();
@ -134,7 +134,7 @@ public final class DynamicBinaryDictIOUtils {
* Helper method to move a PtNode to the tail of the file.
*/
private static int movePtNode(final OutputStream destination,
final Ver3DictUpdater dictUpdater, final PtNodeInfo info,
final Ver2DictUpdater dictUpdater, final PtNodeInfo info,
final int nodeArrayOriginAddress, final int oldNodeAddress,
final FormatOptions formatOptions) throws IOException {
final DictBuffer dictBuffer = dictUpdater.getDictBuffer();
@ -151,7 +151,7 @@ public final class DynamicBinaryDictIOUtils {
}
@SuppressWarnings("unused")
private static void updateForwardLink(final Ver3DictUpdater dictUpdater,
private static void updateForwardLink(final Ver2DictUpdater dictUpdater,
final int nodeArrayOriginAddress, final int newNodeArrayAddress,
final FormatOptions formatOptions) {
final DictBuffer dictBuffer = dictUpdater.getDictBuffer();
@ -199,7 +199,7 @@ public final class DynamicBinaryDictIOUtils {
final int length, final int flags, final int frequency, final int parentAddress,
final ArrayList<WeightedString> shortcutTargets,
final ArrayList<PendingAttribute> bigrams, final OutputStream destination,
final Ver3DictUpdater dictUpdater, final int oldPtNodeArrayOrigin,
final Ver2DictUpdater dictUpdater, final int oldPtNodeArrayOrigin,
final int oldPtNodeOrigin, final FormatOptions formatOptions) throws IOException {
int size = 0;
final int newPtNodeOrigin = fileEndAddress + 1;
@ -252,7 +252,7 @@ public final class DynamicBinaryDictIOUtils {
// TODO: Support batch insertion.
// TODO: Remove @UsedForTesting once UserHistoryDictionary is implemented by BinaryDictionary.
@UsedForTesting
public static void insertWord(final Ver3DictUpdater dictUpdater,
public static void insertWord(final Ver2DictUpdater dictUpdater,
final OutputStream destination, final String word, final int frequency,
final ArrayList<WeightedString> bigramStrings,
final ArrayList<WeightedString> shortcuts, final boolean isNotAWord,

4
java/src/com/android/inputmethod/latin/makedict/FormatSpec.java
View File

@ -401,7 +401,7 @@ public final class FormatSpec {
if (dictFile.isDirectory()) {
return new Ver4DictDecoder(dictFile, bufferType);
} else if (dictFile.isFile()) {
return new Ver3DictDecoder(dictFile, bufferType);
return new Ver2DictDecoder(dictFile, bufferType);
}
return null;
}
@ -411,7 +411,7 @@ public final class FormatSpec {
if (dictFile.isDirectory()) {
return new Ver4DictDecoder(dictFile, factory);
} else if (dictFile.isFile()) {
return new Ver3DictDecoder(dictFile, factory);
return new Ver2DictDecoder(dictFile, factory);
}
return null;
}

17
java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java → java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java
View File

@ -34,18 +34,11 @@ import java.util.ArrayList;
import java.util.Arrays;
/**
* An implementation of DictDecoder for version 3 binary dictionary.
* An implementation of DictDecoder for version 2 binary dictionary.
*/
@UsedForTesting
public class Ver3DictDecoder extends AbstractDictDecoder {
private static final String TAG = Ver3DictDecoder.class.getSimpleName();
static {
JniUtils.loadNativeLibrary();
}
// TODO: implement something sensical instead of just a phony method
private static native int doNothing();
public class Ver2DictDecoder extends AbstractDictDecoder {
private static final String TAG = Ver2DictDecoder.class.getSimpleName();
protected static class PtNodeReader extends AbstractDictDecoder.PtNodeReader {
private static int readFrequency(final DictBuffer dictBuffer) {
@ -57,7 +50,7 @@ public class Ver3DictDecoder extends AbstractDictDecoder {
private final DictionaryBufferFactory mBufferFactory;
protected DictBuffer mDictBuffer;
/* package */ Ver3DictDecoder(final File file, final int factoryFlag) {
/* package */ Ver2DictDecoder(final File file, final int factoryFlag) {
mDictionaryBinaryFile = file;
mDictBuffer = null;
@ -72,7 +65,7 @@ public class Ver3DictDecoder extends AbstractDictDecoder {
}
}
/* package */ Ver3DictDecoder(final File file, final DictionaryBufferFactory factory) {
/* package */ Ver2DictDecoder(final File file, final DictionaryBufferFactory factory) {
mDictionaryBinaryFile = file;
mBufferFactory = factory;
}

8
java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java → java/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java
View File

@ -31,16 +31,16 @@ import java.util.ArrayList;
import java.util.Iterator;
/**
* An implementation of DictEncoder for version 3 binary dictionary.
* An implementation of DictEncoder for version 2 binary dictionary.
*/
public class Ver3DictEncoder implements DictEncoder {
public class Ver2DictEncoder implements DictEncoder {
private final File mDictFile;
private OutputStream mOutStream;
private byte[] mBuffer;
private int mPosition;
public Ver3DictEncoder(final File dictFile) {
public Ver2DictEncoder(final File dictFile) {
mDictFile = dictFile;
mOutStream = null;
mBuffer = null;
@ -49,7 +49,7 @@ public class Ver3DictEncoder implements DictEncoder {
// This constructor is used only by BinaryDictOffdeviceUtilsTests.
// If you want to use this in the production code, you should consider keeping consistency of
// the interface of Ver3DictDecoder by using factory.
public Ver3DictEncoder(final OutputStream outStream) {
public Ver2DictEncoder(final OutputStream outStream) {
mDictFile = null;
mOutStream = outStream;
}

6
java/src/com/android/inputmethod/latin/makedict/Ver3DictUpdater.java → java/src/com/android/inputmethod/latin/makedict/Ver2DictUpdater.java
View File

@ -27,14 +27,14 @@ import java.io.OutputStream;
import java.util.ArrayList;
/**
* An implementation of DictUpdater for version 3 binary dictionary.
* An implementation of DictUpdater for version 2 binary dictionary.
*/
@UsedForTesting
public class Ver3DictUpdater extends Ver3DictDecoder implements DictUpdater {
public class Ver2DictUpdater extends Ver2DictDecoder implements DictUpdater {
private OutputStream mOutStream;
@UsedForTesting
public Ver3DictUpdater(final File dictFile, final int factoryType) {
public Ver2DictUpdater(final File dictFile, final int factoryType) {
// DictUpdater must have an updatable DictBuffer.
super(dictFile, ((factoryType & MASK_DICTBUFFER) == USE_BYTEARRAY)
? USE_BYTEARRAY : USE_WRITABLE_BYTEBUFFER);

421
java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java
View File

@ -1,4 +1,3 @@
/*
/*
* Copyright (C) 2013 The Android Open Source Project
*
@ -18,25 +17,15 @@
package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding;
import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
import com.android.inputmethod.latin.BinaryDictionary;
import com.android.inputmethod.latin.Dictionary;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.FileUtils;
import com.android.inputmethod.latin.utils.LocaleUtils;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
/**
* An implementation of DictEncoder for version 4 binary dictionary.
@ -44,197 +33,19 @@ import java.util.Iterator;
@UsedForTesting
public class Ver4DictEncoder implements DictEncoder {
private final File mDictPlacedDir;
private byte[] mTrieBuf;
private int mTriePos;
private OutputStream mTrieOutStream;
private OutputStream mHeaderOutStream;
private OutputStream mFreqOutStream;
private OutputStream mUnigramTimestampOutStream;
private OutputStream mTerminalAddressTableOutStream;
private File mDictDir;
private String mBaseFilename;
private BigramContentWriter mBigramWriter;
private ShortcutContentWriter mShortcutWriter;
@UsedForTesting
public Ver4DictEncoder(final File dictPlacedDir) {
mDictPlacedDir = dictPlacedDir;
}
private static class BigramContentWriter extends SparseTableContentWriter {
private final boolean mWriteTimestamp;
public BigramContentWriter(final String name, final int initialCapacity,
final File baseDir, final boolean writeTimestamp) {
super(name + FormatSpec.BIGRAM_FILE_EXTENSION, initialCapacity,
FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE, baseDir,
getContentFilenames(name, writeTimestamp), getContentIds(writeTimestamp));
mWriteTimestamp = writeTimestamp;
}
private static String[] getContentFilenames(final String name,
final boolean writeTimestamp) {
final String[] contentFilenames;
if (writeTimestamp) {
contentFilenames = new String[] { name + FormatSpec.BIGRAM_FILE_EXTENSION,
name + FormatSpec.BIGRAM_FILE_EXTENSION };
} else {
contentFilenames = new String[] { name + FormatSpec.BIGRAM_FILE_EXTENSION };
}
return contentFilenames;
}
private static String[] getContentIds(final boolean writeTimestamp) {
final String[] contentIds;
if (writeTimestamp) {
contentIds = new String[] { FormatSpec.BIGRAM_FREQ_CONTENT_ID,
FormatSpec.BIGRAM_TIMESTAMP_CONTENT_ID };
} else {
contentIds = new String[] { FormatSpec.BIGRAM_FREQ_CONTENT_ID };
}
return contentIds;
}
public void writeBigramsForOneWord(final int terminalId, final int bigramCount,
final Iterator<WeightedString> bigramIterator, final FusionDictionary dict)
throws IOException {
write(FormatSpec.BIGRAM_FREQ_CONTENT_INDEX, terminalId,
new SparseTableContentWriterInterface() {
// TODO: This builds a FusionDictionary first and iterates it to add words to the binary
// dictionary. However, it is possible to just add words directly to the binary dictionary
// instead.
// In the long run, when we stop supporting version 2, FusionDictionary will become deprecated
// and we can remove it. Then we'll be able to just call BinaryDictionary directly.
@Override
public void write(final OutputStream outStream) throws IOException {
writeBigramsForOneWordInternal(outStream, bigramIterator, dict);
}});
if (mWriteTimestamp) {
write(FormatSpec.BIGRAM_TIMESTAMP_CONTENT_INDEX, terminalId,
new SparseTableContentWriterInterface() {
@Override
public void write(final OutputStream outStream) throws IOException {
initBigramTimestampsCountersAndLevelsForOneWordInternal(outStream,
bigramCount);
}});
}
}
private void writeBigramsForOneWordInternal(final OutputStream outStream,
final Iterator<WeightedString> bigramIterator, final FusionDictionary dict)
throws IOException {
while (bigramIterator.hasNext()) {
final WeightedString bigram = bigramIterator.next();
final PtNode target =
FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord);
final int unigramFrequencyForThisWord = target.mFrequency;
final int bigramFlags = BinaryDictEncoderUtils.makeBigramFlags(
bigramIterator.hasNext(), 0, bigram.mFrequency,
unigramFrequencyForThisWord, bigram.mWord);
BinaryDictEncoderUtils.writeUIntToStream(outStream, bigramFlags,
FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE);
BinaryDictEncoderUtils.writeUIntToStream(outStream, target.mTerminalId,
FormatSpec.PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE);
}
}
private void initBigramTimestampsCountersAndLevelsForOneWordInternal(
final OutputStream outStream, final int bigramCount) throws IOException {
for (int i = 0; i < bigramCount; ++i) {
// TODO: Figure out what initial values should be.
BinaryDictEncoderUtils.writeUIntToStream(outStream, 0 /* value */,
FormatSpec.BIGRAM_TIMESTAMP_SIZE);
BinaryDictEncoderUtils.writeUIntToStream(outStream, 0 /* value */,
FormatSpec.BIGRAM_COUNTER_SIZE);
BinaryDictEncoderUtils.writeUIntToStream(outStream, 0 /* value */,
FormatSpec.BIGRAM_LEVEL_SIZE);
}
}
}
private static class ShortcutContentWriter extends SparseTableContentWriter {
public ShortcutContentWriter(final String name, final int initialCapacity,
final File baseDir) {
super(name + FormatSpec.SHORTCUT_FILE_EXTENSION, initialCapacity,
FormatSpec.SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE, baseDir,
new String[] { name + FormatSpec.SHORTCUT_FILE_EXTENSION },
new String[] { FormatSpec.SHORTCUT_CONTENT_ID });
}
public void writeShortcutForOneWord(final int terminalId,
final Iterator<WeightedString> shortcutIterator) throws IOException {
write(FormatSpec.SHORTCUT_CONTENT_INDEX, terminalId,
new SparseTableContentWriterInterface() {
@Override
public void write(final OutputStream outStream) throws IOException {
writeShortcutForOneWordInternal(outStream, shortcutIterator);
}
});
}
private void writeShortcutForOneWordInternal(final OutputStream outStream,
final Iterator<WeightedString> shortcutIterator) throws IOException {
while (shortcutIterator.hasNext()) {
final WeightedString target = shortcutIterator.next();
final int shortcutFlags = BinaryDictEncoderUtils.makeShortcutFlags(
shortcutIterator.hasNext(), target.mFrequency);
BinaryDictEncoderUtils.writeUIntToStream(outStream, shortcutFlags,
FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE);
CharEncoding.writeString(outStream, target.mWord);
}
}
}
private void openStreams(final FormatOptions formatOptions, final DictionaryOptions dictOptions)
throws FileNotFoundException, IOException {
final FileHeader header = new FileHeader(0, dictOptions, formatOptions);
mBaseFilename = header.getId() + "." + header.getVersion();
mDictDir = new File(mDictPlacedDir, mBaseFilename);
final File trieFile = new File(mDictDir, mBaseFilename + FormatSpec.TRIE_FILE_EXTENSION);
final File headerFile = new File(mDictDir,
mBaseFilename + FormatSpec.HEADER_FILE_EXTENSION);
final File freqFile = new File(mDictDir, mBaseFilename + FormatSpec.FREQ_FILE_EXTENSION);
final File timestampFile = new File(mDictDir,
mBaseFilename + FormatSpec.UNIGRAM_TIMESTAMP_FILE_EXTENSION);
final File terminalAddressTableFile = new File(mDictDir,
mBaseFilename + FormatSpec.TERMINAL_ADDRESS_TABLE_FILE_EXTENSION);
if (!mDictDir.isDirectory()) {
if (mDictDir.exists()) {
FileUtils.deleteRecursively(mDictDir);
}
mDictDir.mkdirs();
}
mTrieOutStream = new FileOutputStream(trieFile);
mHeaderOutStream = new FileOutputStream(headerFile);
mFreqOutStream = new FileOutputStream(freqFile);
mTerminalAddressTableOutStream = new FileOutputStream(terminalAddressTableFile);
if (formatOptions.mHasTimestamp) {
mUnigramTimestampOutStream = new FileOutputStream(timestampFile);
}
}
private void close() throws IOException {
try {
if (mTrieOutStream != null) {
mTrieOutStream.close();
}
if (mHeaderOutStream != null) {
mHeaderOutStream.close();
}
if (mFreqOutStream != null) {
mFreqOutStream.close();
}
if (mTerminalAddressTableOutStream != null) {
mTerminalAddressTableOutStream.close();
}
if (mUnigramTimestampOutStream != null) {
mUnigramTimestampOutStream.close();
}
} finally {
mTrieOutStream = null;
mHeaderOutStream = null;
mFreqOutStream = null;
mTerminalAddressTableOutStream = null;
}
}
@Override
public void writeDictionary(final FusionDictionary dict, final FormatOptions formatOptions)
public void writeDictionary(FusionDictionary dict, FormatOptions formatOptions)
throws IOException, UnsupportedFormatException {
if (formatOptions.mVersion != FormatSpec.VERSION4) {
throw new UnsupportedFormatException("File header has a wrong version number : "
@ -243,208 +54,70 @@ public class Ver4DictEncoder implements DictEncoder {
if (!mDictPlacedDir.isDirectory()) {
throw new UnsupportedFormatException("Given path is not a directory.");
}
if (mTrieOutStream == null) {
openStreams(formatOptions, dict.mOptions);
if (!BinaryDictionary.createEmptyDictFile(mDictPlacedDir.getAbsolutePath(),
FormatSpec.VERSION4, dict.mOptions.mAttributes)) {
throw new IOException("Cannot create dictionary file");
}
BinaryDictEncoderUtils.writeDictionaryHeader(mHeaderOutStream, dict, formatOptions);
MakedictLog.i("Flattening the tree...");
ArrayList<PtNodeArray> flatNodes = BinaryDictEncoderUtils.flattenTree(dict.mRootNodeArray);
int terminalCount = 0;
final ArrayList<PtNode> nodes = CollectionUtils.newArrayList();
for (final PtNodeArray array : flatNodes) {
for (final PtNode node : array.mData) {
if (node.isTerminal()) {
nodes.add(node);
node.mTerminalId = terminalCount++;
final BinaryDictionary binaryDict = new BinaryDictionary(mDictPlacedDir.getAbsolutePath(),
0l, mDictPlacedDir.length(), true /* useFullEditDistance */,
LocaleUtils.constructLocaleFromString(dict.mOptions.mAttributes.get(
FormatSpec.FileHeader.DICTIONARY_LOCALE_ATTRIBUTE)),
Dictionary.TYPE_USER /* Dictionary type. Does not matter for us */,
true /* isUpdatable */);
if (!binaryDict.isValidDictionary()) {
// Somehow createEmptyDictFile returned true, but the file was not created correctly
throw new IOException("Cannot create dictionary file");
}
for (final Word word : dict) {
// TODO: switch to addMultipleDictionaryEntries when they support shortcuts
if (null == word.mShortcutTargets || word.mShortcutTargets.isEmpty()) {
binaryDict.addUnigramWord(word.mWord, word.mFrequency,
null /* shortcutTarget */, 0 /* shortcutProbability */,
word.mIsNotAWord, word.mIsBlacklistEntry, 0 /* timestamp */);
} else {
for (final WeightedString shortcutTarget : word.mShortcutTargets) {
binaryDict.addUnigramWord(word.mWord, word.mFrequency,
shortcutTarget.mWord, shortcutTarget.mFrequency,
word.mIsNotAWord, word.mIsBlacklistEntry, 0 /* timestamp */);
}
}
if (binaryDict.needsToRunGC(true /* mindsBlockByGC */)) {
binaryDict.flushWithGC();
}
Collections.sort(nodes, new Comparator<PtNode>() {
@Override
public int compare(final PtNode lhs, final PtNode rhs) {
if (lhs.mFrequency != rhs.mFrequency) {
return lhs.mFrequency < rhs.mFrequency ? -1 : 1;
}
if (lhs.mTerminalId < rhs.mTerminalId) return -1;
if (lhs.mTerminalId > rhs.mTerminalId) return 1;
return 0;
for (final Word word0 : dict) {
if (null == word0.mBigrams) continue;
for (final WeightedString word1 : word0.mBigrams) {
binaryDict.addBigramWords(word0.mWord, word1.mWord, word1.mFrequency,
0 /* timestamp */);
}
});
int count = 0;
for (final PtNode node : nodes) {
node.mTerminalId = count++;
if (binaryDict.needsToRunGC(true /* mindsBlockByGC */)) {
binaryDict.flushWithGC();
}
MakedictLog.i("Computing addresses...");
BinaryDictEncoderUtils.computeAddresses(dict, flatNodes, formatOptions);
if (MakedictLog.DBG) BinaryDictEncoderUtils.checkFlatPtNodeArrayList(flatNodes);
writeTerminalData(flatNodes, terminalCount);
if (formatOptions.mHasTimestamp) {
initUnigramTimestamps(terminalCount);
}
mBigramWriter = new BigramContentWriter(mBaseFilename, terminalCount, mDictDir,
formatOptions.mHasTimestamp);
writeBigrams(flatNodes, dict);
mShortcutWriter = new ShortcutContentWriter(mBaseFilename, terminalCount, mDictDir);
writeShortcuts(flatNodes);
final PtNodeArray lastNodeArray = flatNodes.get(flatNodes.size() - 1);
final int bufferSize = lastNodeArray.mCachedAddressAfterUpdate + lastNodeArray.mCachedSize;
mTrieBuf = new byte[bufferSize];
MakedictLog.i("Writing file...");
for (PtNodeArray nodeArray : flatNodes) {
BinaryDictEncoderUtils.writePlacedPtNodeArray(dict, this, nodeArray, formatOptions);
}
if (MakedictLog.DBG) {
BinaryDictEncoderUtils.showStatistics(flatNodes);
MakedictLog.i("has " + terminalCount + " terminals.");
}
mTrieOutStream.write(mTrieBuf);
MakedictLog.i("Done");
close();
binaryDict.flushWithGC();
binaryDict.close();
}
@Override
public void setPosition(int position) {
if (mTrieBuf == null || position < 0 || position > mTrieBuf.length) return;
mTriePos = position;
}
@Override
public int getPosition() {
return mTriePos;
return 0;
}
@Override
public void writePtNodeCount(int ptNodeCount) {
final int countSize = BinaryDictIOUtils.getPtNodeCountSize(ptNodeCount);
// ptNodeCount must fit on one byte or two bytes.
// Please see comments in FormatSpec
if (countSize != 1 && countSize != 2) {
throw new RuntimeException("Strange size from getPtNodeCountSize : " + countSize);
}
final int encodedPtNodeCount = (countSize == 2) ?
(ptNodeCount | FormatSpec.LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE_FLAG) : ptNodeCount;
mTriePos = BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, mTriePos, encodedPtNodeCount,
countSize);
}
private void writePtNodeFlags(final PtNode ptNode, final FormatOptions formatOptions) {
final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions);
mTriePos = BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, mTriePos,
BinaryDictEncoderUtils.makePtNodeFlags(ptNode, childrenPos, formatOptions),
FormatSpec.PTNODE_FLAGS_SIZE);
}
private void writeParentPosition(int parentPos, final PtNode ptNode,
final FormatOptions formatOptions) {
if (parentPos != FormatSpec.NO_PARENT_ADDRESS) {
parentPos -= ptNode.mCachedAddressAfterUpdate;
}
mTriePos = BinaryDictEncoderUtils.writeParentAddress(mTrieBuf, mTriePos, parentPos,
formatOptions);
}
private void writeCharacters(final int[] characters, final boolean hasSeveralChars) {
mTriePos = CharEncoding.writeCharArray(characters, mTrieBuf, mTriePos);
if (hasSeveralChars) {
mTrieBuf[mTriePos++] = FormatSpec.PTNODE_CHARACTERS_TERMINATOR;
}
}
private void writeTerminalId(final int terminalId) {
mTriePos = BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, mTriePos, terminalId,
FormatSpec.PTNODE_TERMINAL_ID_SIZE);
}
private void writeChildrenPosition(PtNode ptNode, FormatOptions formatOptions) {
final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions);
if (formatOptions.supportsDynamicUpdate()) {
mTriePos += BinaryDictEncoderUtils.writeSignedChildrenPosition(mTrieBuf,
mTriePos, childrenPos);
} else {
mTriePos += BinaryDictEncoderUtils.writeChildrenPosition(mTrieBuf,
mTriePos, childrenPos);
}
}
private void writeBigrams(final ArrayList<PtNodeArray> flatNodes, final FusionDictionary dict)
throws IOException {
mBigramWriter.openStreams();
for (final PtNodeArray nodeArray : flatNodes) {
for (final PtNode ptNode : nodeArray.mData) {
if (ptNode.mBigrams != null) {
mBigramWriter.writeBigramsForOneWord(ptNode.mTerminalId, ptNode.mBigrams.size(),
ptNode.mBigrams.iterator(), dict);
}
}
}
mBigramWriter.closeStreams();
}
private void writeShortcuts(final ArrayList<PtNodeArray> flatNodes) throws IOException {
mShortcutWriter.openStreams();
for (final PtNodeArray nodeArray : flatNodes) {
for (final PtNode ptNode : nodeArray.mData) {
if (ptNode.mShortcutTargets != null && !ptNode.mShortcutTargets.isEmpty()) {
mShortcutWriter.writeShortcutForOneWord(ptNode.mTerminalId,
ptNode.mShortcutTargets.iterator());
}
}
}
mShortcutWriter.closeStreams();
}
@Override
public void writeForwardLinkAddress(int forwardLinkAddress) {
mTriePos = BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, mTriePos,
forwardLinkAddress, FormatSpec.FORWARD_LINK_ADDRESS_SIZE);
}
@Override
public void writePtNode(final PtNode ptNode, final int parentPosition,
final FormatOptions formatOptions, final FusionDictionary dict) {
writePtNodeFlags(ptNode, formatOptions);
writeParentPosition(parentPosition, ptNode, formatOptions);
writeCharacters(ptNode.mChars, ptNode.hasSeveralChars());
if (ptNode.isTerminal()) {
writeTerminalId(ptNode.mTerminalId);
}
writeChildrenPosition(ptNode, formatOptions);
}
private void writeTerminalData(final ArrayList<PtNodeArray> flatNodes,
final int terminalCount) throws IOException {
final byte[] freqBuf = new byte[terminalCount * FormatSpec.FREQUENCY_AND_FLAGS_SIZE];
final byte[] terminalAddressTableBuf =
new byte[terminalCount * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE];
for (final PtNodeArray nodeArray : flatNodes) {
for (final PtNode ptNode : nodeArray.mData) {
if (ptNode.isTerminal()) {
BinaryDictEncoderUtils.writeUIntToBuffer(freqBuf,
ptNode.mTerminalId * FormatSpec.FREQUENCY_AND_FLAGS_SIZE,
ptNode.mFrequency, FormatSpec.FREQUENCY_AND_FLAGS_SIZE);
BinaryDictEncoderUtils.writeUIntToBuffer(terminalAddressTableBuf,
ptNode.mTerminalId * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE,
ptNode.mCachedAddressAfterUpdate,
FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE);
}
}
}
mFreqOutStream.write(freqBuf);
mTerminalAddressTableOutStream.write(terminalAddressTableBuf);
}
private void initUnigramTimestamps(final int terminalCount) throws IOException {
// Initial value of time stamps for each word is 0.
final byte[] unigramTimestampBuf =
new byte[terminalCount * FormatSpec.UNIGRAM_TIMESTAMP_SIZE];
mUnigramTimestampOutStream.write(unigramTimestampBuf);
public void writePtNode(
PtNode ptNode, int parentPosition, FormatOptions formatOptions, FusionDictionary dict) {
}
}

86
native/jni/Android.mk
View File

@ -39,87 +39,7 @@ endif # TARGET_ARCH
# To suppress compiler warnings for unused variables/functions used for debug features etc.
LOCAL_CFLAGS += -Wno-unused-parameter -Wno-unused-function
LATIN_IME_JNI_SRC_FILES := \
com_android_inputmethod_keyboard_ProximityInfo.cpp \
com_android_inputmethod_latin_BinaryDictionary.cpp \
com_android_inputmethod_latin_DicTraverseSession.cpp \
com_android_inputmethod_latin_makedict_Ver3DictDecoder.cpp \
jni_common.cpp
LATIN_IME_CORE_SRC_FILES := \
suggest/core/suggest.cpp \
$(addprefix suggest/core/dicnode/, \
dic_node.cpp \
dic_node_utils.cpp \
dic_nodes_cache.cpp) \
$(addprefix suggest/core/dictionary/, \
bigram_dictionary.cpp \
bloom_filter.cpp \
dictionary.cpp \
digraph_utils.cpp \
error_type_utils.cpp \
multi_bigram_map.cpp \
unigram_property.cpp) \
$(addprefix suggest/core/layout/, \
additional_proximity_chars.cpp \
proximity_info.cpp \
proximity_info_params.cpp \
proximity_info_state.cpp \
proximity_info_state_utils.cpp) \
suggest/core/policy/weighting.cpp \
suggest/core/session/dic_traverse_session.cpp \
$(addprefix suggest/policyimpl/dictionary/, \
header/header_policy.cpp \
header/header_read_write_utils.cpp \
shortcut/shortcut_list_reading_utils.cpp \
structure/dictionary_structure_with_buffer_policy_factory.cpp) \
$(addprefix suggest/policyimpl/dictionary/bigram/, \
bigram_list_read_write_utils.cpp \
ver4_bigram_list_policy.cpp) \
$(addprefix suggest/policyimpl/dictionary/structure/pt_common/, \
dynamic_pt_gc_event_listeners.cpp \
dynamic_pt_reading_helper.cpp \
dynamic_pt_reading_utils.cpp \
dynamic_pt_updating_helper.cpp \
dynamic_pt_writing_utils.cpp) \
$(addprefix suggest/policyimpl/dictionary/structure/v2/, \
patricia_trie_policy.cpp \
patricia_trie_reading_utils.cpp) \
$(addprefix suggest/policyimpl/dictionary/structure/v4/, \
ver4_dict_buffers.cpp \
ver4_dict_constants.cpp \
ver4_patricia_trie_node_reader.cpp \
ver4_patricia_trie_node_writer.cpp \
ver4_patricia_trie_policy.cpp \
ver4_patricia_trie_reading_utils.cpp \
ver4_patricia_trie_writing_helper.cpp) \
$(addprefix suggest/policyimpl/dictionary/structure/v4/content/, \
bigram_dict_content.cpp \
probability_dict_content.cpp \
shortcut_dict_content.cpp \
sparse_table_dict_content.cpp \
terminal_position_lookup_table.cpp) \
$(addprefix suggest/policyimpl/dictionary/utils/, \
buffer_with_extendable_buffer.cpp \
byte_array_utils.cpp \
dict_file_writing_utils.cpp \
file_utils.cpp \
forgetting_curve_utils.cpp \
format_utils.cpp \
mmapped_buffer.cpp \
sparse_table.cpp) \
suggest/policyimpl/gesture/gesture_suggest_policy_factory.cpp \
$(addprefix suggest/policyimpl/typing/, \
scoring_params.cpp \
typing_scoring.cpp \
typing_suggest_policy.cpp \
typing_traversal.cpp \
typing_weighting.cpp) \
$(addprefix utils/, \
autocorrection_threshold_utils.cpp \
char_utils.cpp \
log_utils.cpp \
time_keeper.cpp)
include $(LOCAL_PATH)/NativeFileList.mk
LOCAL_SRC_FILES := \
$(LATIN_IME_JNI_SRC_FILES) \
@ -172,6 +92,4 @@ LOCAL_LDFLAGS += -ldl
include $(BUILD_SHARED_LIBRARY)
#################### Clean up the tmp vars
LATIN_IME_CORE_SRC_FILES :=
LATIN_IME_JNI_SRC_FILES :=
LATIN_IME_SRC_DIR :=
include $(LOCAL_PATH)/CleanupNativeFileList.mk

17
native/jni/CleanupNativeFileList.mk Normal file
View File

@ -0,0 +1,17 @@
# Copyright (C) 2013 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
LATIN_IME_CORE_SRC_FILES :=
LATIN_IME_JNI_SRC_FILES :=
LATIN_IME_SRC_DIR :=

94
native/jni/NativeFileList.mk Normal file
View File

@ -0,0 +1,94 @@
# Copyright (C) 2013 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
LATIN_IME_JNI_SRC_FILES := \
com_android_inputmethod_keyboard_ProximityInfo.cpp \
com_android_inputmethod_latin_BinaryDictionary.cpp \
com_android_inputmethod_latin_DicTraverseSession.cpp \
jni_common.cpp
LATIN_IME_CORE_SRC_FILES := \
suggest/core/suggest.cpp \
$(addprefix suggest/core/dicnode/, \
dic_node.cpp \
dic_node_utils.cpp \
dic_nodes_cache.cpp) \
$(addprefix suggest/core/dictionary/, \
bigram_dictionary.cpp \
bloom_filter.cpp \
dictionary.cpp \
digraph_utils.cpp \
error_type_utils.cpp \
multi_bigram_map.cpp \
unigram_property.cpp) \
$(addprefix suggest/core/layout/, \
additional_proximity_chars.cpp \
proximity_info.cpp \
proximity_info_params.cpp \
proximity_info_state.cpp \
proximity_info_state_utils.cpp) \
suggest/core/policy/weighting.cpp \
suggest/core/session/dic_traverse_session.cpp \
$(addprefix suggest/policyimpl/dictionary/, \
header/header_policy.cpp \
header/header_read_write_utils.cpp \
shortcut/shortcut_list_reading_utils.cpp \
structure/dictionary_structure_with_buffer_policy_factory.cpp) \
$(addprefix suggest/policyimpl/dictionary/bigram/, \
bigram_list_read_write_utils.cpp \
ver4_bigram_list_policy.cpp) \
$(addprefix suggest/policyimpl/dictionary/structure/pt_common/, \
dynamic_pt_gc_event_listeners.cpp \
dynamic_pt_reading_helper.cpp \
dynamic_pt_reading_utils.cpp \
dynamic_pt_updating_helper.cpp \
dynamic_pt_writing_utils.cpp) \
$(addprefix suggest/policyimpl/dictionary/structure/v2/, \
patricia_trie_policy.cpp \
patricia_trie_reading_utils.cpp) \
$(addprefix suggest/policyimpl/dictionary/structure/v4/, \
ver4_dict_buffers.cpp \
ver4_dict_constants.cpp \
ver4_patricia_trie_node_reader.cpp \
ver4_patricia_trie_node_writer.cpp \
ver4_patricia_trie_policy.cpp \
ver4_patricia_trie_reading_utils.cpp \
ver4_patricia_trie_writing_helper.cpp) \
$(addprefix suggest/policyimpl/dictionary/structure/v4/content/, \
bigram_dict_content.cpp \
probability_dict_content.cpp \
shortcut_dict_content.cpp \
sparse_table_dict_content.cpp \
terminal_position_lookup_table.cpp) \
$(addprefix suggest/policyimpl/dictionary/utils/, \
buffer_with_extendable_buffer.cpp \
byte_array_utils.cpp \
dict_file_writing_utils.cpp \
file_utils.cpp \
forgetting_curve_utils.cpp \
format_utils.cpp \
mmapped_buffer.cpp \
sparse_table.cpp) \
suggest/policyimpl/gesture/gesture_suggest_policy_factory.cpp \
$(addprefix suggest/policyimpl/typing/, \
scoring_params.cpp \
typing_scoring.cpp \
typing_suggest_policy.cpp \
typing_traversal.cpp \
typing_weighting.cpp) \
$(addprefix utils/, \
autocorrection_threshold_utils.cpp \
char_utils.cpp \
log_utils.cpp \
time_keeper.cpp)

47
native/jni/com_android_inputmethod_latin_makedict_Ver3DictDecoder.cpp
View File

@ -1,47 +0,0 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#define LOG_TAG "LatinIME: jni: Ver3DictDecoder"
#include "com_android_inputmethod_latin_makedict_Ver3DictDecoder.h"
#include "defines.h"
#include "jni.h"
#include "jni_common.h"
namespace latinime {
static int latinime_Ver3DictDecoder_doNothing(JNIEnv *env, jclass clazz) {
// This is a phony method for test - it does nothing. It just returns some value
// unlikely to be in memory by chance for testing purposes.
// TODO: remove this method.
return 2097;
}
static const JNINativeMethod sMethods[] = {
{
// TODO: remove this entry when we have one useful method in here
const_cast<char *>("doNothing"),
const_cast<char *>("()I"),
reinterpret_cast<void *>(latinime_Ver3DictDecoder_doNothing)
},
};
int register_Ver3DictDecoder(JNIEnv *env) {
const char *const kClassPathName =
"com/android/inputmethod/latin/makedict/Ver3DictDecoder";
return registerNativeMethods(env, kClassPathName, sMethods, NELEMS(sMethods));
}
} // namespace latinime

9
native/jni/jni_common.cpp
View File

@ -18,12 +18,9 @@
#include "jni_common.h"
#ifndef HOST_TOOL
#include "com_android_inputmethod_keyboard_ProximityInfo.h"
#include "com_android_inputmethod_latin_BinaryDictionary.h"
#include "com_android_inputmethod_latin_DicTraverseSession.h"
#endif
#include "com_android_inputmethod_latin_makedict_Ver3DictDecoder.h"
#include "defines.h"
/*
@ -41,7 +38,6 @@ jint JNI_OnLoad(JavaVM *vm, void *reserved) {
AKLOGE("ERROR: JNIEnv is invalid");
return -1;
}
#ifndef HOST_TOOL
if (!latinime::register_BinaryDictionary(env)) {
AKLOGE("ERROR: BinaryDictionary native registration failed");
return -1;
@ -54,11 +50,6 @@ jint JNI_OnLoad(JavaVM *vm, void *reserved) {
AKLOGE("ERROR: ProximityInfo native registration failed");
return -1;
}
#endif
if (!latinime::register_Ver3DictDecoder(env)) {
AKLOGE("ERROR: Ver3DictDecoder native registration failed");
return -1;
}
/* success -- return valid version number */
return JNI_VERSION_1_6;
}

9
native/jni/src/defines.h
View File

@ -87,12 +87,21 @@ AK_FORCE_INLINE static int intArrayToCharArray(const int *const source, const in
}
#if defined(FLAG_DO_PROFILE) || defined(FLAG_DBG)
#if defined(__ANDROID__)
#include <android/log.h>
#endif // defined(__ANDROID__)
#ifndef LOG_TAG
#define LOG_TAG "LatinIME: "
#endif // LOG_TAG
#if defined(HOST_TOOL)
#include <stdio.h>
#define AKLOGE(fmt, ...) printf(fmt "\n", ##__VA_ARGS__)
#define AKLOGI(fmt, ...) printf(fmt "\n", ##__VA_ARGS__)
#else // defined(HOST_TOOL)
#define AKLOGE(fmt, ...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, fmt, ##__VA_ARGS__)
#define AKLOGI(fmt, ...) __android_log_print(ANDROID_LOG_INFO, LOG_TAG, fmt, ##__VA_ARGS__)
#endif // defined(HOST_TOOL)
#define DUMP_RESULT(words, frequencies) do { dumpResult(words, frequencies); } while (0)
#define DUMP_WORD(word, length) do { dumpWord(word, length); } while (0)

3
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp
View File

@ -33,8 +33,7 @@ const char *const Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION =
".shortcut_index_shortcut";
// Version 4 dictionary size is implicitly limited to 8MB due to 3-byte offsets.
// TODO: Make MAX_DICTIONARY_SIZE 8MB.
const int Ver4DictConstants::MAX_DICTIONARY_SIZE = 2 * 1024 * 1024;
const int Ver4DictConstants::MAX_DICTIONARY_SIZE = 8 * 1024 * 1024;
// Extended region size, which is not GCed region size in dict file + additional buffer size, is
// limited to 1MB to prevent from inefficient traversing.
const int Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE = 1 * 1024 * 1024;

13
native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.cpp
View File

@ -53,6 +53,11 @@ namespace latinime {
// Remove a directory and all files in the directory.
/* static */ bool FileUtils::removeDirAndFiles(const char *const dirPath) {
return removeDirAndFiles(dirPath, 5 /* maxTries */);
}
// Remove a directory and all files in the directory, trying up to maxTimes.
/* static */ bool FileUtils::removeDirAndFiles(const char *const dirPath, const int maxTries) {
DIR *const dir = opendir(dirPath);
if (dir == NULL) {
AKLOGE("Cannot open dir %s.", dirPath);
@ -60,7 +65,7 @@ namespace latinime {
}
struct dirent *dirent;
while ((dirent = readdir(dir)) != NULL) {
if (dirent->d_type != DT_REG) {
if (dirent->d_type == DT_DIR) {
continue;
}
const int filePathBufSize = getFilePathBufSize(dirPath, dirent->d_name);
@ -74,9 +79,15 @@ namespace latinime {
}
closedir(dir);
if (remove(dirPath) != 0) {
if (maxTries > 0) {
// On NFS, deleting files sometimes creates new files. I'm not sure what the
// correct way of dealing with this is, but for the time being, this seems to work.
removeDirAndFiles(dirPath, maxTries - 1);
} else {
AKLOGE("Cannot remove directory %s.", dirPath);
return false;
}
}
return true;
}

2
native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.h
View File

@ -53,6 +53,8 @@ class FileUtils {
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(FileUtils);
static bool removeDirAndFiles(const char *const dirPath, const int maxTries);
};
} // namespace latinime
#endif /* LATINIME_FILE_UTILS_H */

5
tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
View File

@ -119,13 +119,12 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
attributeMap.put(FormatSpec.FileHeader.HAS_HISTORICAL_INFO_ATTRIBUTE,
FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
final String headerFileName = file.getName() + FormatSpec.HEADER_FILE_EXTENSION;
if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
FormatSpec.VERSION4, attributeMap)) {
return file;
} else {
throw new IOException("Empty dictionary " + file.getAbsolutePath() + " "
+ headerFileName + " cannot be created.");
throw new IOException("Empty dictionary " + file.getAbsolutePath()
+ " cannot be created.");
}
}

4
tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java
View File

@ -103,7 +103,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
Log.d(TAG, " end address = " + info.mEndAddress);
}
private static void printNode(final Ver3DictDecoder dictDecoder,
private static void printNode(final Ver2DictDecoder dictDecoder,
final FormatSpec.FormatOptions formatOptions) {
final DictBuffer dictBuffer = dictDecoder.getDictBuffer();
Log.d(TAG, "Node at " + dictBuffer.position());
@ -121,7 +121,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
}
@SuppressWarnings("unused")
private static void printBinaryFile(final Ver3DictDecoder dictDecoder)
private static void printBinaryFile(final Ver2DictDecoder dictDecoder)
throws IOException, UnsupportedFormatException {
final FileHeader fileHeader = dictDecoder.readHeader();
final DictBuffer dictBuffer = dictDecoder.getDictBuffer();

4
tests/src/com/android/inputmethod/latin/makedict/BinaryDictUtils.java
View File

@ -65,7 +65,7 @@ public class BinaryDictUtils {
return new Ver4DictEncoder(cacheDir);
} else if (formatOptions.mVersion == FormatSpec.VERSION3
|| formatOptions.mVersion == FormatSpec.VERSION2) {
return new Ver3DictEncoder(file);
return new Ver2DictEncoder(file);
} else {
throw new RuntimeException("The format option has a wrong version : "
+ formatOptions.mVersion);
@ -77,7 +77,7 @@ public class BinaryDictUtils {
if (formatOptions.mVersion == FormatSpec.VERSION4) {
return new Ver4DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER);
} else if (formatOptions.mVersion == FormatSpec.VERSION3) {
return new Ver3DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER);
return new Ver2DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER);
} else {
throw new UnsupportedFormatException("The format option has a wrong version : "
+ formatOptions.mVersion);

10
tests/src/com/android/inputmethod/latin/makedict/Ver3DictDecoderTests.java → tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoderTests.java
View File

@ -32,10 +32,10 @@ import java.io.FileOutputStream;
import java.io.IOException;
/**
* Unit tests for Ver3DictDecoder
* Unit tests for Ver2DictDecoder
*/
public class Ver3DictDecoderTests extends AndroidTestCase {
private static final String TAG = Ver3DictDecoderTests.class.getSimpleName();
public class Ver2DictDecoderTests extends AndroidTestCase {
private static final String TAG = Ver2DictDecoderTests.class.getSimpleName();
private final byte[] data = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
@ -68,7 +68,7 @@ public class Ver3DictDecoderTests extends AndroidTestCase {
}
assertNotNull(testFile);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(testFile, factory);
final Ver2DictDecoder dictDecoder = new Ver2DictDecoder(testFile, factory);
try {
dictDecoder.openDictBuffer();
} catch (Exception e) {
@ -110,7 +110,7 @@ public class Ver3DictDecoderTests extends AndroidTestCase {
Log.e(TAG, "IOException while the creating temporary file", e);
}
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(testFile, factory);
final Ver2DictDecoder dictDecoder = new Ver2DictDecoder(testFile, factory);
// the default return value of getBuffer() must be null.
assertNull("the default return value of getBuffer() is not null",

6
tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java
View File

@ -28,8 +28,8 @@ import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.makedict.Ver3DictDecoder;
import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
import com.android.inputmethod.latin.makedict.Ver2DictDecoder;
import com.android.inputmethod.latin.makedict.Ver2DictEncoder;
import com.android.inputmethod.latin.personalization.UserHistoryDictionaryBigramList;
import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.BigramDictionaryInterface;
import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListener;
@ -146,7 +146,7 @@ public class UserHistoryDictIOUtilsTests extends AndroidTestCase
private void writeDictToFile(final File file,
final UserHistoryDictionaryBigramList bigramList) {
final DictEncoder dictEncoder = new Ver3DictEncoder(file);
final DictEncoder dictEncoder = new Ver2DictEncoder(file);
UserHistoryDictIOUtils.writeDictionary(dictEncoder, this, bigramList, FORMAT_OPTIONS,
HEADER_OPTIONS);
}

19
tools/dicttool/Android.mk
View File

@ -27,11 +27,28 @@ LATINIME_BASE_SOURCE_DIRECTORY := $(LATINIME_LOCAL_DIR)/java/src/com/android/inp
LATINIME_ANNOTATIONS_SOURCE_DIRECTORY := $(LATINIME_BASE_SOURCE_DIRECTORY)/annotations
LATINIME_CORE_SOURCE_DIRECTORY := $(LATINIME_BASE_SOURCE_DIRECTORY)/latin
MAKEDICT_CORE_SOURCE_DIRECTORY := $(LATINIME_CORE_SOURCE_DIRECTORY)/makedict
# Dependencies for Dicttool. Most of these files are needed by BinaryDictionary.java. Note that
# a significant part of the dependencies are mocked in the compat/ directory, with empty or
# nearly-empty implementations, for parts that we don't use in Dicttool.
USED_TARGETTED_UTILS := \
$(LATINIME_CORE_SOURCE_DIRECTORY)/BinaryDictionary.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/DicTraverseSession.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/Dictionary.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/InputPointers.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/LastComposedWord.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/LatinImeLogger.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/SuggestedWords.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/WordComposer.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/settings/NativeSuggestOptions.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/utils/ByteArrayDictBuffer.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/utils/CollectionUtils.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/utils/FileUtils.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/utils/JniUtils.java
$(LATINIME_CORE_SOURCE_DIRECTORY)/utils/JniUtils.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/utils/LocaleUtils.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/utils/ResizableIntArray.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/utils/StringUtils.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/utils/UnigramProperty.java
DICTTOOL_ONDEVICE_TESTS_DIRECTORY := \
$(LATINIME_LOCAL_DIR)/tests/src/com/android/inputmethod/latin/makedict/

14
tools/dicttool/NativeLib.mk
View File

@ -20,12 +20,17 @@ include $(CLEAR_VARS)
LATINIME_DIR_RELATIVE_TO_DICTTOOL := ../..
ifeq ($(FLAG_DBG), true)
$(warning Making debug version of native library)
LOCAL_CFLAGS += -DFLAG_DBG -funwind-tables -fno-inline
endif #FLAG_DBG
ifneq ($(strip $(HOST_JDK_IS_64BIT_VERSION)),)
LOCAL_CFLAGS += -m64
LOCAL_LDFLAGS += -m64
endif #HOST_JDK_IS_64BIT_VERSION
LOCAL_CFLAGS += -DHOST_TOOL -fPIC
LOCAL_CFLAGS += -DHOST_TOOL -fPIC -Wno-deprecated
LOCAL_NO_DEFAULT_COMPILER_FLAGS := true
LATINIME_NATIVE_JNI_DIR := $(LATINIME_DIR_RELATIVE_TO_DICTTOOL)/native/jni
@ -33,11 +38,7 @@ LATINIME_NATIVE_SRC_DIR := $(LATINIME_DIR_RELATIVE_TO_DICTTOOL)/native/jni/src
LOCAL_C_INCLUDES := $(LOCAL_PATH)/$(LATINIME_NATIVE_SRC_DIR)
# Used in jni_common.cpp to avoid registering useless methods.
LATIN_IME_JNI_SRC_FILES := \
com_android_inputmethod_latin_makedict_Ver3DictDecoder.cpp \
jni_common.cpp
LATIN_IME_CORE_SRC_FILES :=
include $(LOCAL_PATH)/$(LATINIME_NATIVE_JNI_DIR)/NativeFileList.mk
LOCAL_SRC_FILES := \
$(addprefix $(LATINIME_NATIVE_JNI_DIR)/, $(LATIN_IME_JNI_SRC_FILES)) \
@ -48,4 +49,5 @@ LOCAL_MODULE := $(LATINIME_HOST_NATIVE_LIBNAME)
include $(BUILD_HOST_SHARED_LIBRARY)
# Clear our private variables
include $(LOCAL_PATH)/$(LATINIME_NATIVE_JNI_DIR)/CleanupNativeFileList.mk
LATINIME_DIR_RELATIVE_TO_DICTTOOL := ../..

14
native/jni/com_android_inputmethod_latin_makedict_Ver3DictDecoder.h → tools/dicttool/compat/android/content/SharedPreferences.java
View File

@ -14,12 +14,10 @@
* limitations under the License.
*/
#ifndef _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_VER3DICTDECODER_H
#define _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_VER3DICTDECODER_H
package android.content;
#include "jni.h"
namespace latinime {
int register_Ver3DictDecoder(JNIEnv *env);
} // namespace latinime
#endif // _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_VER3DICTDECODER_H
public class SharedPreferences {
public interface OnSharedPreferenceChangeListener {
public void onSharedPreferenceChanged(SharedPreferences sharedPreferences, String key);
}
}

20
tools/dicttool/compat/android/graphics/Rect.java Normal file
View File

@ -0,0 +1,20 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.graphics;
public class Rect {
}

107
tools/dicttool/compat/android/text/TextUtils.java Normal file
View File

@ -0,0 +1,107 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.text;
public class TextUtils {
private TextUtils() { /* cannot be instantiated */ }
/**
* Returns true if the string is null or 0-length.
* @param str the string to be examined
* @return true if str is null or zero length
*/
public static boolean isEmpty(CharSequence str) {
if (str == null || str.length() == 0)
return true;
else
return false;
}
/**
* Returns true if a and b are equal, including if they are both null.
* <p><i>Note: In platform versions 1.1 and earlier, this method only worked well if
* both the arguments were instances of String.</i></p>
* @param a first CharSequence to check
* @param b second CharSequence to check
* @return true if a and b are equal
*/
public static boolean equals(CharSequence a, CharSequence b) {
if (a == b) return true;
int length;
if (a != null && b != null && (length = a.length()) == b.length()) {
if (a instanceof String && b instanceof String) {
return a.equals(b);
} else {
for (int i = 0; i < length; i++) {
if (a.charAt(i) != b.charAt(i)) return false;
}
return true;
}
}
return false;
}
/**
* Returns list of multiple {@link CharSequence} joined into a single
* {@link CharSequence} separated by localized delimiter such as ", ".
*
* @hide
*/
public static CharSequence join(Iterable<CharSequence> list) {
final CharSequence delimiter = ", ";
return join(delimiter, list);
}
/**
* Returns a string containing the tokens joined by delimiters.
* @param tokens an array objects to be joined. Strings will be formed from
* the objects by calling object.toString().
*/
public static String join(CharSequence delimiter, Object[] tokens) {
StringBuilder sb = new StringBuilder();
boolean firstTime = true;
for (Object token: tokens) {
if (firstTime) {
firstTime = false;
} else {
sb.append(delimiter);
}
sb.append(token);
}
return sb.toString();
}
/**
* Returns a string containing the tokens joined by delimiters.
* @param tokens an array objects to be joined. Strings will be formed from
* the objects by calling object.toString().
*/
public static String join(CharSequence delimiter, Iterable tokens) {
StringBuilder sb = new StringBuilder();
boolean firstTime = true;
for (Object token: tokens) {
if (firstTime) {
firstTime = false;
} else {
sb.append(delimiter);
}
sb.append(token);
}
return sb.toString();
}
}

14
tools/dicttool/compat/android/util/Log.java
View File

@ -25,13 +25,19 @@ public class Log {
public static void d(final String tag, final String message) {
System.out.println(tag + " : " + message);
}
public static void d(final String tag, final String message, final Throwable e) {
System.out.println(tag + " : " + message + " : " + e);
public static void d(final String tag, final String message, final Throwable t) {
System.out.println(tag + " : " + message + " : " + t);
}
public static void e(final String tag, final String message) {
d(tag, message);
}
public static void e(final String tag, final String message, final Throwable e) {
d(tag, message, e);
public static void e(final String tag, final String message, final Throwable t) {
d(tag, message, t);
}
public static void w(final String tag, final String message) {
d(tag, message);
}
public static void w(final String tag, final String message, final Throwable t) {
d(tag, message, t);
}
}

21
tools/dicttool/compat/android/view/inputmethod/CompletionInfo.java Normal file
View File

@ -0,0 +1,21 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.view.inputmethod;
public class CompletionInfo {
public final String getText() { return ""; }
}

20
tools/dicttool/compat/android/view/inputmethod/EditorInfo.java Normal file
View File

@ -0,0 +1,20 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.view.inputmethod;
public class EditorInfo {
}

24
tools/dicttool/compat/com/android/inputmethod/keyboard/Key.java Normal file
View File

@ -0,0 +1,24 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.inputmethod.keyboard;
public class Key {
public final int getX() { return 0; }
public final int getY() { return 0; }
public final int getWidth() { return 0; }
public final int getHeight() { return 0; }
}

22
tools/dicttool/compat/com/android/inputmethod/keyboard/Keyboard.java Normal file
View File

@ -0,0 +1,22 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.inputmethod.keyboard;
public class Keyboard {
private final Key KEY = new Key();
public final Key getKey(final int i) { return KEY; }
}

28
tools/dicttool/compat/com/android/inputmethod/keyboard/ProximityInfo.java Normal file
View File

@ -0,0 +1,28 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.inputmethod.keyboard;
public class ProximityInfo {
public long getNativeProximityInfo() { return 0l; }
private static native long setProximityInfoNative(String locale,
int displayWidth, int displayHeight, int gridWidth, int gridHeight,
int mostCommonKeyWidth, int mostCommonKeyHeight, int[] proximityCharsArray,
int keyCount, int[] keyXCoordinates, int[] keyYCoordinates, int[] keyWidths,
int[] keyHeights, int[] keyCharCodes, float[] sweetSpotCenterXs,
float[] sweetSpotCenterYs, float[] sweetSpotRadii);
private static native void releaseProximityInfoNative(long nativeProximityInfo);
}

20
tools/dicttool/compat/com/android/inputmethod/latin/LatinIME.java Normal file
View File

@ -0,0 +1,20 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.inputmethod.latin;
public class LatinIME {
}

21
tools/dicttool/compat/com/android/inputmethod/latin/settings/AdditionalFeaturesSettingUtils.java Normal file
View File

@ -0,0 +1,21 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.inputmethod.latin.settings;
public class AdditionalFeaturesSettingUtils {
public static final int ADDITIONAL_FEATURES_SETTINGS_SIZE = 0;
}

23
tools/dicttool/compat/com/android/inputmethod/latin/settings/SettingsValues.java Normal file
View File

@ -0,0 +1,23 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.inputmethod.latin.settings;
public class SettingsValues {
public boolean isWordCodePoint(final int code) {
return Character.isLetter(code);
}
}

4
tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java
View File

@ -23,7 +23,7 @@ import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.MakedictLog;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
import com.android.inputmethod.latin.makedict.Ver2DictEncoder;
import com.android.inputmethod.latin.makedict.Ver4DictEncoder;
import java.io.BufferedWriter;
@ -361,7 +361,7 @@ public class DictionaryMaker {
if (version == FormatSpec.VERSION4) {
dictEncoder = new Ver4DictEncoder(outputFile);
} else {
dictEncoder = new Ver3DictEncoder(outputFile);
dictEncoder = new Ver2DictEncoder(outputFile);
}
dictEncoder.writeDictionary(dict, formatOptions);
}

4
tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java
View File

@ -24,7 +24,7 @@ import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
import com.android.inputmethod.latin.makedict.Ver2DictEncoder;
import junit.framework.TestCase;
@ -58,7 +58,7 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase {
Compress.getCompressedStream(
Compress.getCompressedStream(
new BufferedOutputStream(new FileOutputStream(dst)))));
final DictEncoder dictEncoder = new Ver3DictEncoder(out);
final DictEncoder dictEncoder = new Ver2DictEncoder(out);
dictEncoder.writeDictionary(dict, new FormatOptions(2, false));
// Test for an actually compressed dictionary and its contents