Have dicttool use the native library to generate v4 dicts.

Yay !

Change-Id: Iea8ced9e81031b9ab7eff05ad9ef7215be248de9

java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java

@@ -21,7 +21,7 @@ import android.util.Log;
 
 import com.android.inputmethod.latin.makedict.DictEncoder;
 import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
-import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
+import com.android.inputmethod.latin.makedict.Ver2DictEncoder;
 
 import java.io.File;
 import java.io.IOException;
@@ -64,7 +64,7 @@ abstract public class AbstractDictionaryWriter {
         final String tempFilePath = file.getAbsolutePath() + ".temp";
         final File tempFile = new File(tempFilePath);
         try {
-            final DictEncoder dictEncoder = new Ver3DictEncoder(tempFile);
+            final DictEncoder dictEncoder = new Ver2DictEncoder(tempFile);
             writeDictionary(dictEncoder, attributeMap);
             tempFile.renameTo(file);
         } catch (IOException e) {

java/src/com/android/inputmethod/latin/BinaryDictionary.java

@@ -367,6 +367,7 @@ public final class BinaryDictionary extends Dictionary {
     public static class LanguageModelParam {
         public final int[] mWord0;
         public final int[] mWord1;
+        // TODO: this needs to be a list of shortcuts
         public final int[] mShortcutTarget;
         public final int mUnigramProbability;
         public final int mBigramProbability;
@@ -375,7 +376,7 @@ public final class BinaryDictionary extends Dictionary {
         public final boolean mIsBlacklisted;
         public final int mTimestamp;
 
-        // Constructor for unigram.
+        // Constructor for unigram. TODO: support shortcuts
         public LanguageModelParam(final String word, final int unigramProbability,
                 final int timestamp) {
             mWord0 = null;

java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java

@@ -55,7 +55,7 @@ public final class DynamicBinaryDictIOUtils {
      * @param newParentAddress the absolute address of the parent.
      * @param formatOptions file format options.
      */
-    private static void updateParentAddress(final Ver3DictUpdater dictUpdater,
+    private static void updateParentAddress(final Ver2DictUpdater dictUpdater,
             final int ptNodeOriginAddress, final int newParentAddress,
             final FormatOptions formatOptions) {
         final DictBuffer dictBuffer = dictUpdater.getDictBuffer();
@@ -88,7 +88,7 @@ public final class DynamicBinaryDictIOUtils {
      * @param newParentAddress the address to be written.
      * @param formatOptions file format options.
      */
-    private static void updateParentAddresses(final Ver3DictUpdater dictUpdater,
+    private static void updateParentAddresses(final Ver2DictUpdater dictUpdater,
             final int ptNodeOriginAddress, final int newParentAddress,
             final FormatOptions formatOptions) {
         final int originalPosition = dictUpdater.getPosition();
@@ -114,7 +114,7 @@ public final class DynamicBinaryDictIOUtils {
      * @param newChildrenAddress the absolute address of the child.
      * @param formatOptions file format options.
      */
-    private static void updateChildrenAddress(final Ver3DictUpdater dictUpdater,
+    private static void updateChildrenAddress(final Ver2DictUpdater dictUpdater,
             final int ptNodeOriginAddress, final int newChildrenAddress,
             final FormatOptions formatOptions) {
         final DictBuffer dictBuffer = dictUpdater.getDictBuffer();
@@ -134,7 +134,7 @@ public final class DynamicBinaryDictIOUtils {
      * Helper method to move a PtNode to the tail of the file.
      */
     private static int movePtNode(final OutputStream destination,
-            final Ver3DictUpdater dictUpdater, final PtNodeInfo info,
+            final Ver2DictUpdater dictUpdater, final PtNodeInfo info,
             final int nodeArrayOriginAddress, final int oldNodeAddress,
             final FormatOptions formatOptions) throws IOException {
         final DictBuffer dictBuffer = dictUpdater.getDictBuffer();
@@ -151,7 +151,7 @@ public final class DynamicBinaryDictIOUtils {
     }
 
     @SuppressWarnings("unused")
-    private static void updateForwardLink(final Ver3DictUpdater dictUpdater,
+    private static void updateForwardLink(final Ver2DictUpdater dictUpdater,
             final int nodeArrayOriginAddress, final int newNodeArrayAddress,
             final FormatOptions formatOptions) {
         final DictBuffer dictBuffer = dictUpdater.getDictBuffer();
@@ -199,7 +199,7 @@ public final class DynamicBinaryDictIOUtils {
             final int length, final int flags, final int frequency, final int parentAddress,
             final ArrayList<WeightedString> shortcutTargets,
             final ArrayList<PendingAttribute> bigrams, final OutputStream destination,
-            final Ver3DictUpdater dictUpdater, final int oldPtNodeArrayOrigin,
+            final Ver2DictUpdater dictUpdater, final int oldPtNodeArrayOrigin,
             final int oldPtNodeOrigin, final FormatOptions formatOptions) throws IOException {
         int size = 0;
         final int newPtNodeOrigin = fileEndAddress + 1;
@@ -252,7 +252,7 @@ public final class DynamicBinaryDictIOUtils {
     // TODO: Support batch insertion.
     // TODO: Remove @UsedForTesting once UserHistoryDictionary is implemented by BinaryDictionary.
     @UsedForTesting
-    public static void insertWord(final Ver3DictUpdater dictUpdater,
+    public static void insertWord(final Ver2DictUpdater dictUpdater,
             final OutputStream destination, final String word, final int frequency,
             final ArrayList<WeightedString> bigramStrings,
             final ArrayList<WeightedString> shortcuts, final boolean isNotAWord,

java/src/com/android/inputmethod/latin/makedict/FormatSpec.java

@@ -401,7 +401,7 @@ public final class FormatSpec {
         if (dictFile.isDirectory()) {
             return new Ver4DictDecoder(dictFile, bufferType);
         } else if (dictFile.isFile()) {
-            return new Ver3DictDecoder(dictFile, bufferType);
+            return new Ver2DictDecoder(dictFile, bufferType);
         }
         return null;
     }
@@ -411,7 +411,7 @@ public final class FormatSpec {
         if (dictFile.isDirectory()) {
             return new Ver4DictDecoder(dictFile, factory);
         } else if (dictFile.isFile()) {
-            return new Ver3DictDecoder(dictFile, factory);
+            return new Ver2DictDecoder(dictFile, factory);
         }
         return null;
     }

java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java

@@ -34,18 +34,11 @@ import java.util.ArrayList;
 import java.util.Arrays;
 
 /**
- * An implementation of DictDecoder for version 3 binary dictionary.
+ * An implementation of DictDecoder for version 2 binary dictionary.
  */
 @UsedForTesting
-public class Ver3DictDecoder extends AbstractDictDecoder {
+public class Ver2DictDecoder extends AbstractDictDecoder {
-    private static final String TAG = Ver3DictDecoder.class.getSimpleName();
+    private static final String TAG = Ver2DictDecoder.class.getSimpleName();
-
-    static {
-        JniUtils.loadNativeLibrary();
-    }
-
-    // TODO: implement something sensical instead of just a phony method
-    private static native int doNothing();
 
     protected static class PtNodeReader extends AbstractDictDecoder.PtNodeReader {
         private static int readFrequency(final DictBuffer dictBuffer) {
@@ -57,7 +50,7 @@ public class Ver3DictDecoder extends AbstractDictDecoder {
     private final DictionaryBufferFactory mBufferFactory;
     protected DictBuffer mDictBuffer;
 
-    /* package */ Ver3DictDecoder(final File file, final int factoryFlag) {
+    /* package */ Ver2DictDecoder(final File file, final int factoryFlag) {
         mDictionaryBinaryFile = file;
         mDictBuffer = null;
 
@@ -72,7 +65,7 @@ public class Ver3DictDecoder extends AbstractDictDecoder {
         }
     }
 
-    /* package */ Ver3DictDecoder(final File file, final DictionaryBufferFactory factory) {
+    /* package */ Ver2DictDecoder(final File file, final DictionaryBufferFactory factory) {
         mDictionaryBinaryFile = file;
         mBufferFactory = factory;
     }
@@ -166,7 +159,7 @@ public class Ver3DictDecoder extends AbstractDictDecoder {
         final ArrayList<PendingAttribute> bigrams;
         if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) {
             bigrams = new ArrayList<PendingAttribute>();
-            addressPointer += PtNodeReader.readBigramAddresses(mDictBuffer, bigrams, 
+            addressPointer += PtNodeReader.readBigramAddresses(mDictBuffer, bigrams,
                     addressPointer);
             if (bigrams.size() >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) {
                 throw new RuntimeException("Too many bigrams in a PtNode (" + bigrams.size()

java/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java

@@ -31,16 +31,16 @@ import java.util.ArrayList;
 import java.util.Iterator;
 
 /**
- * An implementation of DictEncoder for version 3 binary dictionary.
+ * An implementation of DictEncoder for version 2 binary dictionary.
  */
-public class Ver3DictEncoder implements DictEncoder {
+public class Ver2DictEncoder implements DictEncoder {
 
     private final File mDictFile;
     private OutputStream mOutStream;
     private byte[] mBuffer;
     private int mPosition;
 
-    public Ver3DictEncoder(final File dictFile) {
+    public Ver2DictEncoder(final File dictFile) {
         mDictFile = dictFile;
         mOutStream = null;
         mBuffer = null;
@@ -49,7 +49,7 @@ public class Ver3DictEncoder implements DictEncoder {
     // This constructor is used only by BinaryDictOffdeviceUtilsTests.
     // If you want to use this in the production code, you should consider keeping consistency of
     // the interface of Ver3DictDecoder by using factory.
-    public Ver3DictEncoder(final OutputStream outStream) {
+    public Ver2DictEncoder(final OutputStream outStream) {
         mDictFile = null;
         mOutStream = outStream;
     }

java/src/com/android/inputmethod/latin/makedict/Ver2DictUpdater.java

@@ -27,14 +27,14 @@ import java.io.OutputStream;
 import java.util.ArrayList;
 
 /**
- * An implementation of DictUpdater for version 3 binary dictionary.
+ * An implementation of DictUpdater for version 2 binary dictionary.
  */
 @UsedForTesting
-public class Ver3DictUpdater extends Ver3DictDecoder implements DictUpdater {
+public class Ver2DictUpdater extends Ver2DictDecoder implements DictUpdater {
     private OutputStream mOutStream;
 
     @UsedForTesting
-    public Ver3DictUpdater(final File dictFile, final int factoryType) {
+    public Ver2DictUpdater(final File dictFile, final int factoryType) {
         // DictUpdater must have an updatable DictBuffer.
         super(dictFile, ((factoryType & MASK_DICTBUFFER) == USE_BYTEARRAY)
                 ? USE_BYTEARRAY : USE_WRITABLE_BYTEBUFFER);

java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java

@@ -1,4 +1,3 @@
-/*
 /*
  * Copyright (C) 2013 The Android Open Source Project
  *
@@ -18,25 +17,15 @@
 package com.android.inputmethod.latin.makedict;
 
 import com.android.inputmethod.annotations.UsedForTesting;
-import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding;
+import com.android.inputmethod.latin.BinaryDictionary;
-import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
+import com.android.inputmethod.latin.Dictionary;
 import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
-import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
 import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
-import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
 import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
-import com.android.inputmethod.latin.utils.CollectionUtils;
+import com.android.inputmethod.latin.utils.LocaleUtils;
-import com.android.inputmethod.latin.utils.FileUtils;
 
 import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Iterator;
 
 /**
  * An implementation of DictEncoder for version 4 binary dictionary.
@@ -44,197 +33,19 @@ import java.util.Iterator;
 @UsedForTesting
 public class Ver4DictEncoder implements DictEncoder {
     private final File mDictPlacedDir;
-    private byte[] mTrieBuf;
-    private int mTriePos;
-    private OutputStream mTrieOutStream;
-    private OutputStream mHeaderOutStream;
-    private OutputStream mFreqOutStream;
-    private OutputStream mUnigramTimestampOutStream;
-    private OutputStream mTerminalAddressTableOutStream;
-    private File mDictDir;
-    private String mBaseFilename;
-    private BigramContentWriter mBigramWriter;
-    private ShortcutContentWriter mShortcutWriter;
 
     @UsedForTesting
     public Ver4DictEncoder(final File dictPlacedDir) {
         mDictPlacedDir = dictPlacedDir;
     }
 
-    private static class BigramContentWriter extends SparseTableContentWriter {
+    // TODO: This builds a FusionDictionary first and iterates it to add words to the binary
-        private final boolean mWriteTimestamp;
+    // dictionary. However, it is possible to just add words directly to the binary dictionary
-
+    // instead.
-        public BigramContentWriter(final String name, final int initialCapacity,
+    // In the long run, when we stop supporting version 2, FusionDictionary will become deprecated
-                final File baseDir, final boolean writeTimestamp) {
+    // and we can remove it. Then we'll be able to just call BinaryDictionary directly.
-            super(name + FormatSpec.BIGRAM_FILE_EXTENSION, initialCapacity,
-                    FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE, baseDir,
-                    getContentFilenames(name, writeTimestamp), getContentIds(writeTimestamp));
-            mWriteTimestamp = writeTimestamp;
-        }
-
-        private static String[] getContentFilenames(final String name,
-                final boolean writeTimestamp) {
-            final String[] contentFilenames;
-            if (writeTimestamp) {
-                contentFilenames = new String[] { name + FormatSpec.BIGRAM_FILE_EXTENSION,
-                        name + FormatSpec.BIGRAM_FILE_EXTENSION };
-            } else {
-                contentFilenames = new String[] { name + FormatSpec.BIGRAM_FILE_EXTENSION };
-            }
-            return contentFilenames;
-        }
-
-        private static String[] getContentIds(final boolean writeTimestamp) {
-            final String[] contentIds;
-            if (writeTimestamp) {
-                contentIds = new String[] { FormatSpec.BIGRAM_FREQ_CONTENT_ID,
-                        FormatSpec.BIGRAM_TIMESTAMP_CONTENT_ID };
-            } else {
-                contentIds = new String[] { FormatSpec.BIGRAM_FREQ_CONTENT_ID };
-            }
-            return contentIds;
-        }
-
-        public void writeBigramsForOneWord(final int terminalId, final int bigramCount,
-                final Iterator<WeightedString> bigramIterator, final FusionDictionary dict)
-                        throws IOException {
-            write(FormatSpec.BIGRAM_FREQ_CONTENT_INDEX, terminalId,
-                    new SparseTableContentWriterInterface() {
-                        @Override
-                        public void write(final OutputStream outStream) throws IOException {
-                            writeBigramsForOneWordInternal(outStream, bigramIterator, dict);
-                        }});
-            if (mWriteTimestamp) {
-                write(FormatSpec.BIGRAM_TIMESTAMP_CONTENT_INDEX, terminalId,
-                        new SparseTableContentWriterInterface() {
-                            @Override
-                            public void write(final OutputStream outStream) throws IOException {
-                                initBigramTimestampsCountersAndLevelsForOneWordInternal(outStream,
-                                        bigramCount);
-                            }});
-            }
-        }
-
-        private void writeBigramsForOneWordInternal(final OutputStream outStream,
-                final Iterator<WeightedString> bigramIterator, final FusionDictionary dict)
-                        throws IOException {
-            while (bigramIterator.hasNext()) {
-                final WeightedString bigram = bigramIterator.next();
-                final PtNode target =
-                        FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord);
-                final int unigramFrequencyForThisWord = target.mFrequency;
-                final int bigramFlags = BinaryDictEncoderUtils.makeBigramFlags(
-                        bigramIterator.hasNext(), 0, bigram.mFrequency,
-                        unigramFrequencyForThisWord, bigram.mWord);
-                BinaryDictEncoderUtils.writeUIntToStream(outStream, bigramFlags,
-                        FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE);
-                BinaryDictEncoderUtils.writeUIntToStream(outStream, target.mTerminalId,
-                        FormatSpec.PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE);
-            }
-        }
-
-        private void initBigramTimestampsCountersAndLevelsForOneWordInternal(
-                final OutputStream outStream, final int bigramCount) throws IOException {
-            for (int i = 0; i < bigramCount; ++i) {
-                // TODO: Figure out what initial values should be.
-                BinaryDictEncoderUtils.writeUIntToStream(outStream, 0 /* value */,
-                        FormatSpec.BIGRAM_TIMESTAMP_SIZE);
-                BinaryDictEncoderUtils.writeUIntToStream(outStream, 0 /* value */,
-                        FormatSpec.BIGRAM_COUNTER_SIZE);
-                BinaryDictEncoderUtils.writeUIntToStream(outStream, 0 /* value */,
-                        FormatSpec.BIGRAM_LEVEL_SIZE);
-            }
-        }
-    }
-
-    private static class ShortcutContentWriter extends SparseTableContentWriter {
-        public ShortcutContentWriter(final String name, final int initialCapacity,
-                final File baseDir) {
-            super(name + FormatSpec.SHORTCUT_FILE_EXTENSION, initialCapacity,
-                    FormatSpec.SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE, baseDir,
-                    new String[] { name + FormatSpec.SHORTCUT_FILE_EXTENSION },
-                    new String[] { FormatSpec.SHORTCUT_CONTENT_ID });
-        }
-
-        public void writeShortcutForOneWord(final int terminalId,
-                final Iterator<WeightedString> shortcutIterator) throws IOException {
-            write(FormatSpec.SHORTCUT_CONTENT_INDEX, terminalId,
-                    new SparseTableContentWriterInterface() {
-                        @Override
-                        public void write(final OutputStream outStream) throws IOException {
-                            writeShortcutForOneWordInternal(outStream, shortcutIterator);
-                        }
-                    });
-        }
-
-        private void writeShortcutForOneWordInternal(final OutputStream outStream,
-                final Iterator<WeightedString> shortcutIterator) throws IOException {
-            while (shortcutIterator.hasNext()) {
-                final WeightedString target = shortcutIterator.next();
-                final int shortcutFlags = BinaryDictEncoderUtils.makeShortcutFlags(
-                        shortcutIterator.hasNext(), target.mFrequency);
-                BinaryDictEncoderUtils.writeUIntToStream(outStream, shortcutFlags,
-                        FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE);
-                CharEncoding.writeString(outStream, target.mWord);
-            }
-        }
-    }
-
-    private void openStreams(final FormatOptions formatOptions, final DictionaryOptions dictOptions)
-            throws FileNotFoundException, IOException {
-        final FileHeader header = new FileHeader(0, dictOptions, formatOptions);
-        mBaseFilename = header.getId() + "." + header.getVersion();
-        mDictDir = new File(mDictPlacedDir, mBaseFilename);
-        final File trieFile = new File(mDictDir, mBaseFilename + FormatSpec.TRIE_FILE_EXTENSION);
-        final File headerFile = new File(mDictDir,
-                mBaseFilename + FormatSpec.HEADER_FILE_EXTENSION);
-        final File freqFile = new File(mDictDir, mBaseFilename + FormatSpec.FREQ_FILE_EXTENSION);
-        final File timestampFile = new File(mDictDir,
-                mBaseFilename + FormatSpec.UNIGRAM_TIMESTAMP_FILE_EXTENSION);
-        final File terminalAddressTableFile = new File(mDictDir,
-                mBaseFilename + FormatSpec.TERMINAL_ADDRESS_TABLE_FILE_EXTENSION);
-        if (!mDictDir.isDirectory()) {
-            if (mDictDir.exists()) {
-                FileUtils.deleteRecursively(mDictDir);
-            }
-            mDictDir.mkdirs();
-        }
-        mTrieOutStream = new FileOutputStream(trieFile);
-        mHeaderOutStream = new FileOutputStream(headerFile);
-        mFreqOutStream = new FileOutputStream(freqFile);
-        mTerminalAddressTableOutStream = new FileOutputStream(terminalAddressTableFile);
-        if (formatOptions.mHasTimestamp) {
-            mUnigramTimestampOutStream = new FileOutputStream(timestampFile);
-        }
-    }
-
-    private void close() throws IOException {
-        try {
-            if (mTrieOutStream != null) {
-                mTrieOutStream.close();
-            }
-            if (mHeaderOutStream != null) {
-                mHeaderOutStream.close();
-            }
-            if (mFreqOutStream != null) {
-                mFreqOutStream.close();
-            }
-            if (mTerminalAddressTableOutStream != null) {
-                mTerminalAddressTableOutStream.close();
-            }
-            if (mUnigramTimestampOutStream != null) {
-                mUnigramTimestampOutStream.close();
-            }
-        } finally {
-            mTrieOutStream = null;
-            mHeaderOutStream = null;
-            mFreqOutStream = null;
-            mTerminalAddressTableOutStream = null;
-        }
-    }
-
     @Override
-    public void writeDictionary(final FusionDictionary dict, final FormatOptions formatOptions)
+    public void writeDictionary(FusionDictionary dict, FormatOptions formatOptions)
             throws IOException, UnsupportedFormatException {
         if (formatOptions.mVersion != FormatSpec.VERSION4) {
             throw new UnsupportedFormatException("File header has a wrong version number : "
@@ -243,208 +54,70 @@ public class Ver4DictEncoder implements DictEncoder {
         if (!mDictPlacedDir.isDirectory()) {
             throw new UnsupportedFormatException("Given path is not a directory.");
         }
-
+        if (!BinaryDictionary.createEmptyDictFile(mDictPlacedDir.getAbsolutePath(),
-        if (mTrieOutStream == null) {
+                FormatSpec.VERSION4, dict.mOptions.mAttributes)) {
-            openStreams(formatOptions, dict.mOptions);
+            throw new IOException("Cannot create dictionary file");
         }
-
+        final BinaryDictionary binaryDict = new BinaryDictionary(mDictPlacedDir.getAbsolutePath(),
-        BinaryDictEncoderUtils.writeDictionaryHeader(mHeaderOutStream, dict, formatOptions);
+                0l, mDictPlacedDir.length(), true /* useFullEditDistance */,
-
+                LocaleUtils.constructLocaleFromString(dict.mOptions.mAttributes.get(
-        MakedictLog.i("Flattening the tree...");
+                        FormatSpec.FileHeader.DICTIONARY_LOCALE_ATTRIBUTE)),
-        ArrayList<PtNodeArray> flatNodes = BinaryDictEncoderUtils.flattenTree(dict.mRootNodeArray);
+                Dictionary.TYPE_USER /* Dictionary type. Does not matter for us */,
-        int terminalCount = 0;
+                true /* isUpdatable */);
-        final ArrayList<PtNode> nodes = CollectionUtils.newArrayList();
+        if (!binaryDict.isValidDictionary()) {
-        for (final PtNodeArray array : flatNodes) {
+            // Somehow createEmptyDictFile returned true, but the file was not created correctly
-            for (final PtNode node : array.mData) {
+            throw new IOException("Cannot create dictionary file");
-                if (node.isTerminal()) {
+        }
-                    nodes.add(node);
+        for (final Word word : dict) {
-                    node.mTerminalId = terminalCount++;
+            // TODO: switch to addMultipleDictionaryEntries when they support shortcuts
+            if (null == word.mShortcutTargets || word.mShortcutTargets.isEmpty()) {
+                binaryDict.addUnigramWord(word.mWord, word.mFrequency,
+                        null /* shortcutTarget */, 0 /* shortcutProbability */,
+                        word.mIsNotAWord, word.mIsBlacklistEntry, 0 /* timestamp */);
+            } else {
+                for (final WeightedString shortcutTarget : word.mShortcutTargets) {
+                    binaryDict.addUnigramWord(word.mWord, word.mFrequency,
+                            shortcutTarget.mWord, shortcutTarget.mFrequency,
+                            word.mIsNotAWord, word.mIsBlacklistEntry, 0 /* timestamp */);
                 }
             }
-        }
+            if (binaryDict.needsToRunGC(true /* mindsBlockByGC */)) {
-        Collections.sort(nodes, new Comparator<PtNode>() {
+                binaryDict.flushWithGC();
-            @Override
-            public int compare(final PtNode lhs, final PtNode rhs) {
-                if (lhs.mFrequency != rhs.mFrequency) {
-                    return lhs.mFrequency < rhs.mFrequency ? -1 : 1;
-                }
-                if (lhs.mTerminalId < rhs.mTerminalId) return -1;
-                if (lhs.mTerminalId > rhs.mTerminalId) return 1;
-                return 0;
             }
-        });
-        int count = 0;
-        for (final PtNode node : nodes) {
-            node.mTerminalId = count++;
         }
-
+        for (final Word word0 : dict) {
-        MakedictLog.i("Computing addresses...");
+            if (null == word0.mBigrams) continue;
-        BinaryDictEncoderUtils.computeAddresses(dict, flatNodes, formatOptions);
+            for (final WeightedString word1 : word0.mBigrams) {
-        if (MakedictLog.DBG) BinaryDictEncoderUtils.checkFlatPtNodeArrayList(flatNodes);
+                binaryDict.addBigramWords(word0.mWord, word1.mWord, word1.mFrequency,
-
+                        0 /* timestamp */);
-        writeTerminalData(flatNodes, terminalCount);
+            }
-        if (formatOptions.mHasTimestamp) {
+            if (binaryDict.needsToRunGC(true /* mindsBlockByGC */)) {
-            initUnigramTimestamps(terminalCount);
+                binaryDict.flushWithGC();
+            }
         }
-        mBigramWriter = new BigramContentWriter(mBaseFilename, terminalCount, mDictDir,
+        binaryDict.flushWithGC();
-                formatOptions.mHasTimestamp);
+        binaryDict.close();
-        writeBigrams(flatNodes, dict);
-        mShortcutWriter = new ShortcutContentWriter(mBaseFilename, terminalCount, mDictDir);
-        writeShortcuts(flatNodes);
-
-        final PtNodeArray lastNodeArray = flatNodes.get(flatNodes.size() - 1);
-        final int bufferSize = lastNodeArray.mCachedAddressAfterUpdate + lastNodeArray.mCachedSize;
-        mTrieBuf = new byte[bufferSize];
-
-        MakedictLog.i("Writing file...");
-        for (PtNodeArray nodeArray : flatNodes) {
-            BinaryDictEncoderUtils.writePlacedPtNodeArray(dict, this, nodeArray, formatOptions);
-        }
-        if (MakedictLog.DBG) {
-            BinaryDictEncoderUtils.showStatistics(flatNodes);
-            MakedictLog.i("has " + terminalCount + " terminals.");
-        }
-        mTrieOutStream.write(mTrieBuf);
-
-        MakedictLog.i("Done");
-        close();
     }
 
     @Override
     public void setPosition(int position) {
-        if (mTrieBuf == null || position < 0 || position > mTrieBuf.length) return;
-        mTriePos = position;
     }
 
     @Override
     public int getPosition() {
-        return mTriePos;
+        return 0;
     }
 
     @Override
     public void writePtNodeCount(int ptNodeCount) {
-        final int countSize = BinaryDictIOUtils.getPtNodeCountSize(ptNodeCount);
-        // ptNodeCount must fit on one byte or two bytes.
-        // Please see comments in FormatSpec
-        if (countSize != 1 && countSize != 2) {
-            throw new RuntimeException("Strange size from getPtNodeCountSize : " + countSize);
-        }
-        final int encodedPtNodeCount = (countSize == 2) ?
-                (ptNodeCount | FormatSpec.LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE_FLAG) : ptNodeCount;
-        mTriePos = BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, mTriePos, encodedPtNodeCount,
-                countSize);
-    }
-
-    private void writePtNodeFlags(final PtNode ptNode, final FormatOptions formatOptions) {
-        final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions);
-        mTriePos = BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, mTriePos,
-                BinaryDictEncoderUtils.makePtNodeFlags(ptNode, childrenPos, formatOptions),
-                FormatSpec.PTNODE_FLAGS_SIZE);
-    }
-
-    private void writeParentPosition(int parentPos, final PtNode ptNode,
-            final FormatOptions formatOptions) {
-        if (parentPos != FormatSpec.NO_PARENT_ADDRESS) {
-            parentPos -= ptNode.mCachedAddressAfterUpdate;
-        }
-        mTriePos = BinaryDictEncoderUtils.writeParentAddress(mTrieBuf, mTriePos, parentPos,
-                formatOptions);
-    }
-
-    private void writeCharacters(final int[] characters, final boolean hasSeveralChars) {
-        mTriePos = CharEncoding.writeCharArray(characters, mTrieBuf, mTriePos);
-        if (hasSeveralChars) {
-            mTrieBuf[mTriePos++] = FormatSpec.PTNODE_CHARACTERS_TERMINATOR;
-        }
-    }
-
-    private void writeTerminalId(final int terminalId) {
-        mTriePos = BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, mTriePos, terminalId,
-                FormatSpec.PTNODE_TERMINAL_ID_SIZE);
-    }
-
-    private void writeChildrenPosition(PtNode ptNode, FormatOptions formatOptions) {
-        final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions);
-        if (formatOptions.supportsDynamicUpdate()) {
-            mTriePos += BinaryDictEncoderUtils.writeSignedChildrenPosition(mTrieBuf,
-                    mTriePos, childrenPos);
-        } else {
-            mTriePos += BinaryDictEncoderUtils.writeChildrenPosition(mTrieBuf,
-                    mTriePos, childrenPos);
-        }
-    }
-
-    private void writeBigrams(final ArrayList<PtNodeArray> flatNodes, final FusionDictionary dict)
-            throws IOException {
-        mBigramWriter.openStreams();
-        for (final PtNodeArray nodeArray : flatNodes) {
-            for (final PtNode ptNode : nodeArray.mData) {
-                if (ptNode.mBigrams != null) {
-                    mBigramWriter.writeBigramsForOneWord(ptNode.mTerminalId, ptNode.mBigrams.size(),
-                            ptNode.mBigrams.iterator(), dict);
-                }
-            }
-        }
-        mBigramWriter.closeStreams();
-    }
-
-    private void writeShortcuts(final ArrayList<PtNodeArray> flatNodes) throws IOException {
-        mShortcutWriter.openStreams();
-        for (final PtNodeArray nodeArray : flatNodes) {
-            for (final PtNode ptNode : nodeArray.mData) {
-                if (ptNode.mShortcutTargets != null && !ptNode.mShortcutTargets.isEmpty()) {
-                    mShortcutWriter.writeShortcutForOneWord(ptNode.mTerminalId,
-                            ptNode.mShortcutTargets.iterator());
-                }
-            }
-        }
-        mShortcutWriter.closeStreams();
     }
 
     @Override
     public void writeForwardLinkAddress(int forwardLinkAddress) {
-        mTriePos = BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, mTriePos,
-                forwardLinkAddress, FormatSpec.FORWARD_LINK_ADDRESS_SIZE);
     }
 
     @Override
-    public void writePtNode(final PtNode ptNode, final int parentPosition,
+    public void writePtNode(
-            final FormatOptions formatOptions, final FusionDictionary dict) {
+            PtNode ptNode, int parentPosition, FormatOptions formatOptions, FusionDictionary dict) {
-        writePtNodeFlags(ptNode, formatOptions);
-        writeParentPosition(parentPosition, ptNode, formatOptions);
-        writeCharacters(ptNode.mChars, ptNode.hasSeveralChars());
-        if (ptNode.isTerminal()) {
-            writeTerminalId(ptNode.mTerminalId);
-        }
-        writeChildrenPosition(ptNode, formatOptions);
-    }
-
-    private void writeTerminalData(final ArrayList<PtNodeArray> flatNodes,
-          final int terminalCount) throws IOException {
-        final byte[] freqBuf = new byte[terminalCount * FormatSpec.FREQUENCY_AND_FLAGS_SIZE];
-        final byte[] terminalAddressTableBuf =
-                new byte[terminalCount * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE];
-        for (final PtNodeArray nodeArray : flatNodes) {
-            for (final PtNode ptNode : nodeArray.mData) {
-                if (ptNode.isTerminal()) {
-                    BinaryDictEncoderUtils.writeUIntToBuffer(freqBuf,
-                            ptNode.mTerminalId * FormatSpec.FREQUENCY_AND_FLAGS_SIZE,
-                            ptNode.mFrequency, FormatSpec.FREQUENCY_AND_FLAGS_SIZE);
-                    BinaryDictEncoderUtils.writeUIntToBuffer(terminalAddressTableBuf,
-                            ptNode.mTerminalId * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE,
-                            ptNode.mCachedAddressAfterUpdate,
-                            FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE);
-                }
-            }
-        }
-        mFreqOutStream.write(freqBuf);
-        mTerminalAddressTableOutStream.write(terminalAddressTableBuf);
-    }
-
-    private void initUnigramTimestamps(final int terminalCount) throws IOException {
-        // Initial value of time stamps for each word is 0.
-        final byte[] unigramTimestampBuf =
-                new byte[terminalCount * FormatSpec.UNIGRAM_TIMESTAMP_SIZE];
-        mUnigramTimestampOutStream.write(unigramTimestampBuf);
     }
 }

native/jni/Android.mk

@@ -39,87 +39,7 @@ endif # TARGET_ARCH
 # To suppress compiler warnings for unused variables/functions used for debug features etc.
 LOCAL_CFLAGS += -Wno-unused-parameter -Wno-unused-function
 
-LATIN_IME_JNI_SRC_FILES := \
+include $(LOCAL_PATH)/NativeFileList.mk
-    com_android_inputmethod_keyboard_ProximityInfo.cpp \
-    com_android_inputmethod_latin_BinaryDictionary.cpp \
-    com_android_inputmethod_latin_DicTraverseSession.cpp \
-    com_android_inputmethod_latin_makedict_Ver3DictDecoder.cpp \
-    jni_common.cpp
-
-LATIN_IME_CORE_SRC_FILES := \
-    suggest/core/suggest.cpp \
-    $(addprefix suggest/core/dicnode/, \
-        dic_node.cpp \
-        dic_node_utils.cpp \
-        dic_nodes_cache.cpp) \
-    $(addprefix suggest/core/dictionary/, \
-        bigram_dictionary.cpp \
-        bloom_filter.cpp \
-        dictionary.cpp \
-        digraph_utils.cpp \
-        error_type_utils.cpp \
-        multi_bigram_map.cpp \
-        unigram_property.cpp) \
-    $(addprefix suggest/core/layout/, \
-        additional_proximity_chars.cpp \
-        proximity_info.cpp \
-        proximity_info_params.cpp \
-        proximity_info_state.cpp \
-        proximity_info_state_utils.cpp) \
-    suggest/core/policy/weighting.cpp \
-    suggest/core/session/dic_traverse_session.cpp \
-    $(addprefix suggest/policyimpl/dictionary/, \
-        header/header_policy.cpp \
-        header/header_read_write_utils.cpp \
-        shortcut/shortcut_list_reading_utils.cpp \
-        structure/dictionary_structure_with_buffer_policy_factory.cpp) \
-    $(addprefix suggest/policyimpl/dictionary/bigram/, \
-        bigram_list_read_write_utils.cpp \
-        ver4_bigram_list_policy.cpp) \
-    $(addprefix suggest/policyimpl/dictionary/structure/pt_common/, \
-        dynamic_pt_gc_event_listeners.cpp \
-        dynamic_pt_reading_helper.cpp \
-        dynamic_pt_reading_utils.cpp \
-        dynamic_pt_updating_helper.cpp \
-        dynamic_pt_writing_utils.cpp) \
-    $(addprefix suggest/policyimpl/dictionary/structure/v2/, \
-        patricia_trie_policy.cpp \
-        patricia_trie_reading_utils.cpp) \
-    $(addprefix suggest/policyimpl/dictionary/structure/v4/, \
-        ver4_dict_buffers.cpp \
-        ver4_dict_constants.cpp \
-        ver4_patricia_trie_node_reader.cpp \
-        ver4_patricia_trie_node_writer.cpp \
-        ver4_patricia_trie_policy.cpp \
-        ver4_patricia_trie_reading_utils.cpp \
-        ver4_patricia_trie_writing_helper.cpp) \
-    $(addprefix suggest/policyimpl/dictionary/structure/v4/content/, \
-        bigram_dict_content.cpp \
-        probability_dict_content.cpp \
-        shortcut_dict_content.cpp \
-        sparse_table_dict_content.cpp \
-        terminal_position_lookup_table.cpp) \
-    $(addprefix suggest/policyimpl/dictionary/utils/, \
-        buffer_with_extendable_buffer.cpp \
-        byte_array_utils.cpp \
-        dict_file_writing_utils.cpp \
-        file_utils.cpp \
-        forgetting_curve_utils.cpp \
-        format_utils.cpp \
-        mmapped_buffer.cpp \
-        sparse_table.cpp) \
-    suggest/policyimpl/gesture/gesture_suggest_policy_factory.cpp \
-    $(addprefix suggest/policyimpl/typing/, \
-        scoring_params.cpp \
-        typing_scoring.cpp \
-        typing_suggest_policy.cpp \
-        typing_traversal.cpp \
-        typing_weighting.cpp) \
-    $(addprefix utils/, \
-        autocorrection_threshold_utils.cpp \
-        char_utils.cpp \
-        log_utils.cpp \
-        time_keeper.cpp)
 
 LOCAL_SRC_FILES := \
     $(LATIN_IME_JNI_SRC_FILES) \
@@ -172,6 +92,4 @@ LOCAL_LDFLAGS += -ldl
 include $(BUILD_SHARED_LIBRARY)
 
 #################### Clean up the tmp vars
-LATIN_IME_CORE_SRC_FILES :=
+include $(LOCAL_PATH)/CleanupNativeFileList.mk
-LATIN_IME_JNI_SRC_FILES :=
-LATIN_IME_SRC_DIR :=

native/jni/CleanupNativeFileList.mk

@@ -0,0 +1,17 @@
+# Copyright (C) 2013 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+LATIN_IME_CORE_SRC_FILES :=
+LATIN_IME_JNI_SRC_FILES :=
+LATIN_IME_SRC_DIR :=

native/jni/NativeFileList.mk

@@ -0,0 +1,94 @@
+# Copyright (C) 2013 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+LATIN_IME_JNI_SRC_FILES := \
+    com_android_inputmethod_keyboard_ProximityInfo.cpp \
+    com_android_inputmethod_latin_BinaryDictionary.cpp \
+    com_android_inputmethod_latin_DicTraverseSession.cpp \
+    jni_common.cpp
+
+LATIN_IME_CORE_SRC_FILES := \
+    suggest/core/suggest.cpp \
+    $(addprefix suggest/core/dicnode/, \
+        dic_node.cpp \
+        dic_node_utils.cpp \
+        dic_nodes_cache.cpp) \
+    $(addprefix suggest/core/dictionary/, \
+        bigram_dictionary.cpp \
+        bloom_filter.cpp \
+        dictionary.cpp \
+        digraph_utils.cpp \
+        error_type_utils.cpp \
+        multi_bigram_map.cpp \
+        unigram_property.cpp) \
+    $(addprefix suggest/core/layout/, \
+        additional_proximity_chars.cpp \
+        proximity_info.cpp \
+        proximity_info_params.cpp \
+        proximity_info_state.cpp \
+        proximity_info_state_utils.cpp) \
+    suggest/core/policy/weighting.cpp \
+    suggest/core/session/dic_traverse_session.cpp \
+    $(addprefix suggest/policyimpl/dictionary/, \
+        header/header_policy.cpp \
+        header/header_read_write_utils.cpp \
+        shortcut/shortcut_list_reading_utils.cpp \
+        structure/dictionary_structure_with_buffer_policy_factory.cpp) \
+    $(addprefix suggest/policyimpl/dictionary/bigram/, \
+        bigram_list_read_write_utils.cpp \
+        ver4_bigram_list_policy.cpp) \
+    $(addprefix suggest/policyimpl/dictionary/structure/pt_common/, \
+        dynamic_pt_gc_event_listeners.cpp \
+        dynamic_pt_reading_helper.cpp \
+        dynamic_pt_reading_utils.cpp \
+        dynamic_pt_updating_helper.cpp \
+        dynamic_pt_writing_utils.cpp) \
+    $(addprefix suggest/policyimpl/dictionary/structure/v2/, \
+        patricia_trie_policy.cpp \
+        patricia_trie_reading_utils.cpp) \
+    $(addprefix suggest/policyimpl/dictionary/structure/v4/, \
+        ver4_dict_buffers.cpp \
+        ver4_dict_constants.cpp \
+        ver4_patricia_trie_node_reader.cpp \
+        ver4_patricia_trie_node_writer.cpp \
+        ver4_patricia_trie_policy.cpp \
+        ver4_patricia_trie_reading_utils.cpp \
+        ver4_patricia_trie_writing_helper.cpp) \
+    $(addprefix suggest/policyimpl/dictionary/structure/v4/content/, \
+        bigram_dict_content.cpp \
+        probability_dict_content.cpp \
+        shortcut_dict_content.cpp \
+        sparse_table_dict_content.cpp \
+        terminal_position_lookup_table.cpp) \
+    $(addprefix suggest/policyimpl/dictionary/utils/, \
+        buffer_with_extendable_buffer.cpp \
+        byte_array_utils.cpp \
+        dict_file_writing_utils.cpp \
+        file_utils.cpp \
+        forgetting_curve_utils.cpp \
+        format_utils.cpp \
+        mmapped_buffer.cpp \
+        sparse_table.cpp) \
+    suggest/policyimpl/gesture/gesture_suggest_policy_factory.cpp \
+    $(addprefix suggest/policyimpl/typing/, \
+        scoring_params.cpp \
+        typing_scoring.cpp \
+        typing_suggest_policy.cpp \
+        typing_traversal.cpp \
+        typing_weighting.cpp) \
+    $(addprefix utils/, \
+        autocorrection_threshold_utils.cpp \
+        char_utils.cpp \
+        log_utils.cpp \
+        time_keeper.cpp)

native/jni/com_android_inputmethod_latin_makedict_Ver3DictDecoder.cpp

@@ -1,47 +0,0 @@
-/*
- * Copyright (C) 2013 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#define LOG_TAG "LatinIME: jni: Ver3DictDecoder"
-
-#include "com_android_inputmethod_latin_makedict_Ver3DictDecoder.h"
-
-#include "defines.h"
-#include "jni.h"
-#include "jni_common.h"
-
-namespace latinime {
-static int latinime_Ver3DictDecoder_doNothing(JNIEnv *env, jclass clazz) {
-    // This is a phony method for test - it does nothing. It just returns some value
-    // unlikely to be in memory by chance for testing purposes.
-    // TODO: remove this method.
-    return 2097;
-}
-
-static const JNINativeMethod sMethods[] = {
-    {
-        // TODO: remove this entry when we have one useful method in here
-        const_cast<char *>("doNothing"),
-        const_cast<char *>("()I"),
-        reinterpret_cast<void *>(latinime_Ver3DictDecoder_doNothing)
-    },
-};
-
-int register_Ver3DictDecoder(JNIEnv *env) {
-    const char *const kClassPathName =
-            "com/android/inputmethod/latin/makedict/Ver3DictDecoder";
-    return registerNativeMethods(env, kClassPathName, sMethods, NELEMS(sMethods));
-}
-} // namespace latinime

native/jni/jni_common.cpp

@@ -18,12 +18,9 @@
 
 #include "jni_common.h"
 
-#ifndef HOST_TOOL
 #include "com_android_inputmethod_keyboard_ProximityInfo.h"
 #include "com_android_inputmethod_latin_BinaryDictionary.h"
 #include "com_android_inputmethod_latin_DicTraverseSession.h"
-#endif
-#include "com_android_inputmethod_latin_makedict_Ver3DictDecoder.h"
 #include "defines.h"
 
 /*
@@ -41,7 +38,6 @@ jint JNI_OnLoad(JavaVM *vm, void *reserved) {
         AKLOGE("ERROR: JNIEnv is invalid");
         return -1;
     }
-#ifndef HOST_TOOL
     if (!latinime::register_BinaryDictionary(env)) {
         AKLOGE("ERROR: BinaryDictionary native registration failed");
         return -1;
@@ -54,11 +50,6 @@ jint JNI_OnLoad(JavaVM *vm, void *reserved) {
         AKLOGE("ERROR: ProximityInfo native registration failed");
         return -1;
     }
-#endif
-    if (!latinime::register_Ver3DictDecoder(env)) {
-        AKLOGE("ERROR: Ver3DictDecoder native registration failed");
-        return -1;
-    }
     /* success -- return valid version number */
     return JNI_VERSION_1_6;
 }

native/jni/src/defines.h

@@ -87,12 +87,21 @@ AK_FORCE_INLINE static int intArrayToCharArray(const int *const source, const in
 }
 
 #if defined(FLAG_DO_PROFILE) || defined(FLAG_DBG)
+#if defined(__ANDROID__)
 #include <android/log.h>
+#endif // defined(__ANDROID__)
 #ifndef LOG_TAG
 #define LOG_TAG "LatinIME: "
 #endif // LOG_TAG
+
+#if defined(HOST_TOOL)
+#include <stdio.h>
+#define AKLOGE(fmt, ...) printf(fmt "\n", ##__VA_ARGS__)
+#define AKLOGI(fmt, ...) printf(fmt "\n", ##__VA_ARGS__)
+#else // defined(HOST_TOOL)
 #define AKLOGE(fmt, ...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, fmt, ##__VA_ARGS__)
 #define AKLOGI(fmt, ...) __android_log_print(ANDROID_LOG_INFO, LOG_TAG, fmt, ##__VA_ARGS__)
+#endif // defined(HOST_TOOL)
 
 #define DUMP_RESULT(words, frequencies) do { dumpResult(words, frequencies); } while (0)
 #define DUMP_WORD(word, length) do { dumpWord(word, length); } while (0)

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp

@@ -33,8 +33,7 @@ const char *const Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION =
         ".shortcut_index_shortcut";
 
 // Version 4 dictionary size is implicitly limited to 8MB due to 3-byte offsets.
-// TODO: Make MAX_DICTIONARY_SIZE 8MB.
+const int Ver4DictConstants::MAX_DICTIONARY_SIZE = 8 * 1024 * 1024;
-const int Ver4DictConstants::MAX_DICTIONARY_SIZE = 2 * 1024 * 1024;
 // Extended region size, which is not GCed region size in dict file + additional buffer size, is
 // limited to 1MB to prevent from inefficient traversing.
 const int Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE = 1 * 1024 * 1024;

native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.cpp

@@ -53,6 +53,11 @@ namespace latinime {
 
 // Remove a directory and all files in the directory.
 /* static */ bool FileUtils::removeDirAndFiles(const char *const dirPath) {
+    return removeDirAndFiles(dirPath, 5 /* maxTries */);
+}
+
+// Remove a directory and all files in the directory, trying up to maxTimes.
+/* static */ bool FileUtils::removeDirAndFiles(const char *const dirPath, const int maxTries) {
     DIR *const dir = opendir(dirPath);
     if (dir == NULL) {
         AKLOGE("Cannot open dir %s.", dirPath);
@@ -60,7 +65,7 @@ namespace latinime {
     }
     struct dirent *dirent;
     while ((dirent = readdir(dir)) != NULL) {
-        if (dirent->d_type != DT_REG) {
+        if (dirent->d_type == DT_DIR) {
             continue;
         }
         const int filePathBufSize = getFilePathBufSize(dirPath, dirent->d_name);
@@ -74,8 +79,14 @@ namespace latinime {
     }
     closedir(dir);
     if (remove(dirPath) != 0) {
-        AKLOGE("Cannot remove directory %s.", dirPath);
+        if (maxTries > 0) {
-        return false;
+            // On NFS, deleting files sometimes creates new files. I'm not sure what the
+            // correct way of dealing with this is, but for the time being, this seems to work.
+            removeDirAndFiles(dirPath, maxTries - 1);
+        } else {
+            AKLOGE("Cannot remove directory %s.", dirPath);
+            return false;
+        }
     }
     return true;
 }

native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.h

@@ -53,6 +53,8 @@ class FileUtils {
 
  private:
     DISALLOW_IMPLICIT_CONSTRUCTORS(FileUtils);
+
+    static bool removeDirAndFiles(const char *const dirPath, const int maxTries);
 };
 } // namespace latinime
 #endif /* LATINIME_FILE_UTILS_H */

tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java

@@ -119,13 +119,12 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
                 FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
         attributeMap.put(FormatSpec.FileHeader.HAS_HISTORICAL_INFO_ATTRIBUTE,
                 FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
-        final String headerFileName = file.getName() + FormatSpec.HEADER_FILE_EXTENSION;
         if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
                 FormatSpec.VERSION4, attributeMap)) {
             return file;
         } else {
-            throw new IOException("Empty dictionary " + file.getAbsolutePath() + " "
+            throw new IOException("Empty dictionary " + file.getAbsolutePath()
-                    + headerFileName + " cannot be created.");
+                    + " cannot be created.");
         }
     }
 

tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java

@@ -103,7 +103,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
         Log.d(TAG, "    end address = " + info.mEndAddress);
     }
 
-    private static void printNode(final Ver3DictDecoder dictDecoder,
+    private static void printNode(final Ver2DictDecoder dictDecoder,
             final FormatSpec.FormatOptions formatOptions) {
         final DictBuffer dictBuffer = dictDecoder.getDictBuffer();
         Log.d(TAG, "Node at " + dictBuffer.position());
@@ -121,7 +121,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
     }
 
     @SuppressWarnings("unused")
-    private static void printBinaryFile(final Ver3DictDecoder dictDecoder)
+    private static void printBinaryFile(final Ver2DictDecoder dictDecoder)
             throws IOException, UnsupportedFormatException {
         final FileHeader fileHeader = dictDecoder.readHeader();
         final DictBuffer dictBuffer = dictDecoder.getDictBuffer();

tests/src/com/android/inputmethod/latin/makedict/BinaryDictUtils.java

@@ -65,7 +65,7 @@ public class BinaryDictUtils {
             return new Ver4DictEncoder(cacheDir);
         } else if (formatOptions.mVersion == FormatSpec.VERSION3
                 || formatOptions.mVersion == FormatSpec.VERSION2) {
-            return new Ver3DictEncoder(file);
+            return new Ver2DictEncoder(file);
         } else {
             throw new RuntimeException("The format option has a wrong version : "
                     + formatOptions.mVersion);
@@ -77,7 +77,7 @@ public class BinaryDictUtils {
         if (formatOptions.mVersion == FormatSpec.VERSION4) {
             return new Ver4DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER);
         } else if (formatOptions.mVersion == FormatSpec.VERSION3) {
-            return new Ver3DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER);
+            return new Ver2DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER);
         } else {
             throw new UnsupportedFormatException("The format option has a wrong version : "
                     + formatOptions.mVersion);

tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoderTests.java

@@ -32,10 +32,10 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 
 /**
- * Unit tests for Ver3DictDecoder
+ * Unit tests for Ver2DictDecoder
  */
-public class Ver3DictDecoderTests extends AndroidTestCase {
+public class Ver2DictDecoderTests extends AndroidTestCase {
-    private static final String TAG = Ver3DictDecoderTests.class.getSimpleName();
+    private static final String TAG = Ver2DictDecoderTests.class.getSimpleName();
 
     private final byte[] data = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
 
@@ -68,7 +68,7 @@ public class Ver3DictDecoderTests extends AndroidTestCase {
         }
 
         assertNotNull(testFile);
-        final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(testFile, factory);
+        final Ver2DictDecoder dictDecoder = new Ver2DictDecoder(testFile, factory);
         try {
             dictDecoder.openDictBuffer();
         } catch (Exception e) {
@@ -110,7 +110,7 @@ public class Ver3DictDecoderTests extends AndroidTestCase {
             Log.e(TAG, "IOException while the creating temporary file", e);
         }
 
-        final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(testFile, factory);
+        final Ver2DictDecoder dictDecoder = new Ver2DictDecoder(testFile, factory);
 
         // the default return value of getBuffer() must be null.
         assertNull("the default return value of getBuffer() is not null",

tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java

@@ -28,8 +28,8 @@ import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
 import com.android.inputmethod.latin.makedict.FusionDictionary;
 import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
 import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
-import com.android.inputmethod.latin.makedict.Ver3DictDecoder;
+import com.android.inputmethod.latin.makedict.Ver2DictDecoder;
-import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
+import com.android.inputmethod.latin.makedict.Ver2DictEncoder;
 import com.android.inputmethod.latin.personalization.UserHistoryDictionaryBigramList;
 import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.BigramDictionaryInterface;
 import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListener;
@@ -146,7 +146,7 @@ public class UserHistoryDictIOUtilsTests extends AndroidTestCase
 
     private void writeDictToFile(final File file,
             final UserHistoryDictionaryBigramList bigramList) {
-        final DictEncoder dictEncoder = new Ver3DictEncoder(file);
+        final DictEncoder dictEncoder = new Ver2DictEncoder(file);
         UserHistoryDictIOUtils.writeDictionary(dictEncoder, this, bigramList, FORMAT_OPTIONS,
                 HEADER_OPTIONS);
     }

tools/dicttool/Android.mk

@@ -27,11 +27,28 @@ LATINIME_BASE_SOURCE_DIRECTORY := $(LATINIME_LOCAL_DIR)/java/src/com/android/inp
 LATINIME_ANNOTATIONS_SOURCE_DIRECTORY := $(LATINIME_BASE_SOURCE_DIRECTORY)/annotations
 LATINIME_CORE_SOURCE_DIRECTORY := $(LATINIME_BASE_SOURCE_DIRECTORY)/latin
 MAKEDICT_CORE_SOURCE_DIRECTORY := $(LATINIME_CORE_SOURCE_DIRECTORY)/makedict
+
+# Dependencies for Dicttool. Most of these files are needed by BinaryDictionary.java. Note that
+# a significant part of the dependencies are mocked in the compat/ directory, with empty or
+# nearly-empty implementations, for parts that we don't use in Dicttool.
 USED_TARGETTED_UTILS := \
+        $(LATINIME_CORE_SOURCE_DIRECTORY)/BinaryDictionary.java \
+        $(LATINIME_CORE_SOURCE_DIRECTORY)/DicTraverseSession.java \
+        $(LATINIME_CORE_SOURCE_DIRECTORY)/Dictionary.java \
+        $(LATINIME_CORE_SOURCE_DIRECTORY)/InputPointers.java \
+        $(LATINIME_CORE_SOURCE_DIRECTORY)/LastComposedWord.java \
+        $(LATINIME_CORE_SOURCE_DIRECTORY)/LatinImeLogger.java \
+        $(LATINIME_CORE_SOURCE_DIRECTORY)/SuggestedWords.java \
+        $(LATINIME_CORE_SOURCE_DIRECTORY)/WordComposer.java \
+        $(LATINIME_CORE_SOURCE_DIRECTORY)/settings/NativeSuggestOptions.java \
         $(LATINIME_CORE_SOURCE_DIRECTORY)/utils/ByteArrayDictBuffer.java \
         $(LATINIME_CORE_SOURCE_DIRECTORY)/utils/CollectionUtils.java \
         $(LATINIME_CORE_SOURCE_DIRECTORY)/utils/FileUtils.java \
-        $(LATINIME_CORE_SOURCE_DIRECTORY)/utils/JniUtils.java
+        $(LATINIME_CORE_SOURCE_DIRECTORY)/utils/JniUtils.java \
+        $(LATINIME_CORE_SOURCE_DIRECTORY)/utils/LocaleUtils.java \
+        $(LATINIME_CORE_SOURCE_DIRECTORY)/utils/ResizableIntArray.java \
+        $(LATINIME_CORE_SOURCE_DIRECTORY)/utils/StringUtils.java \
+        $(LATINIME_CORE_SOURCE_DIRECTORY)/utils/UnigramProperty.java
 
 DICTTOOL_ONDEVICE_TESTS_DIRECTORY := \
         $(LATINIME_LOCAL_DIR)/tests/src/com/android/inputmethod/latin/makedict/

tools/dicttool/NativeLib.mk

@@ -20,12 +20,17 @@ include $(CLEAR_VARS)
 
 LATINIME_DIR_RELATIVE_TO_DICTTOOL := ../..
 
+ifeq ($(FLAG_DBG), true)
+    $(warning Making debug version of native library)
+    LOCAL_CFLAGS += -DFLAG_DBG -funwind-tables -fno-inline
+endif #FLAG_DBG
+
 ifneq ($(strip $(HOST_JDK_IS_64BIT_VERSION)),)
 LOCAL_CFLAGS += -m64
 LOCAL_LDFLAGS += -m64
 endif #HOST_JDK_IS_64BIT_VERSION
 
-LOCAL_CFLAGS += -DHOST_TOOL -fPIC
+LOCAL_CFLAGS += -DHOST_TOOL -fPIC -Wno-deprecated
 LOCAL_NO_DEFAULT_COMPILER_FLAGS := true
 
 LATINIME_NATIVE_JNI_DIR := $(LATINIME_DIR_RELATIVE_TO_DICTTOOL)/native/jni
@@ -33,11 +38,7 @@ LATINIME_NATIVE_SRC_DIR := $(LATINIME_DIR_RELATIVE_TO_DICTTOOL)/native/jni/src
 LOCAL_C_INCLUDES := $(LOCAL_PATH)/$(LATINIME_NATIVE_SRC_DIR)
 # Used in jni_common.cpp to avoid registering useless methods.
 
-LATIN_IME_JNI_SRC_FILES := \
+include $(LOCAL_PATH)/$(LATINIME_NATIVE_JNI_DIR)/NativeFileList.mk
-    com_android_inputmethod_latin_makedict_Ver3DictDecoder.cpp \
-    jni_common.cpp
-
-LATIN_IME_CORE_SRC_FILES :=
 
 LOCAL_SRC_FILES := \
     $(addprefix $(LATINIME_NATIVE_JNI_DIR)/, $(LATIN_IME_JNI_SRC_FILES)) \
@@ -48,4 +49,5 @@ LOCAL_MODULE := $(LATINIME_HOST_NATIVE_LIBNAME)
 include $(BUILD_HOST_SHARED_LIBRARY)
 
 # Clear our private variables
+include $(LOCAL_PATH)/$(LATINIME_NATIVE_JNI_DIR)/CleanupNativeFileList.mk
 LATINIME_DIR_RELATIVE_TO_DICTTOOL := ../..

tools/dicttool/compat/android/content/SharedPreferences.java

@@ -14,12 +14,10 @@
  * limitations under the License.
  */
 
-#ifndef _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_VER3DICTDECODER_H
+package android.content;
-#define _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_VER3DICTDECODER_H
 
-#include "jni.h"
+public class SharedPreferences {
-
+    public interface OnSharedPreferenceChangeListener {
-namespace latinime {
+        public void onSharedPreferenceChanged(SharedPreferences sharedPreferences, String key);
-int register_Ver3DictDecoder(JNIEnv *env);
+    }
-} // namespace latinime
+}
-#endif // _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_VER3DICTDECODER_H

tools/dicttool/compat/android/graphics/Rect.java

@@ -0,0 +1,20 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.graphics;
+
+public class Rect {
+}

tools/dicttool/compat/android/text/TextUtils.java

@@ -0,0 +1,107 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.text;
+
+public class TextUtils {
+    private TextUtils() { /* cannot be instantiated */ }
+
+    /**
+     * Returns true if the string is null or 0-length.
+     * @param str the string to be examined
+     * @return true if str is null or zero length
+     */
+    public static boolean isEmpty(CharSequence str) {
+        if (str == null || str.length() == 0)
+            return true;
+        else
+            return false;
+    }
+
+    /**
+     * Returns true if a and b are equal, including if they are both null.
+     * <p><i>Note: In platform versions 1.1 and earlier, this method only worked well if
+     * both the arguments were instances of String.</i></p>
+     * @param a first CharSequence to check
+     * @param b second CharSequence to check
+     * @return true if a and b are equal
+     */
+    public static boolean equals(CharSequence a, CharSequence b) {
+        if (a == b) return true;
+        int length;
+        if (a != null && b != null && (length = a.length()) == b.length()) {
+            if (a instanceof String && b instanceof String) {
+                return a.equals(b);
+            } else {
+                for (int i = 0; i < length; i++) {
+                    if (a.charAt(i) != b.charAt(i)) return false;
+                }
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Returns list of multiple {@link CharSequence} joined into a single
+     * {@link CharSequence} separated by localized delimiter such as ", ".
+     *
+     * @hide
+     */
+    public static CharSequence join(Iterable<CharSequence> list) {
+        final CharSequence delimiter = ", ";
+        return join(delimiter, list);
+    }
+
+    /**
+     * Returns a string containing the tokens joined by delimiters.
+     * @param tokens an array objects to be joined. Strings will be formed from
+     *     the objects by calling object.toString().
+     */
+    public static String join(CharSequence delimiter, Object[] tokens) {
+        StringBuilder sb = new StringBuilder();
+        boolean firstTime = true;
+        for (Object token: tokens) {
+            if (firstTime) {
+                firstTime = false;
+            } else {
+                sb.append(delimiter);
+            }
+            sb.append(token);
+        }
+        return sb.toString();
+    }
+
+    /**
+     * Returns a string containing the tokens joined by delimiters.
+     * @param tokens an array objects to be joined. Strings will be formed from
+     *     the objects by calling object.toString().
+     */
+    public static String join(CharSequence delimiter, Iterable tokens) {
+        StringBuilder sb = new StringBuilder();
+        boolean firstTime = true;
+        for (Object token: tokens) {
+            if (firstTime) {
+                firstTime = false;
+            } else {
+                sb.append(delimiter);
+            }
+            sb.append(token);
+        }
+        return sb.toString();
+    }
+
+}

tools/dicttool/compat/android/util/Log.java

@@ -25,13 +25,19 @@ public class Log {
     public static void d(final String tag, final String message) {
         System.out.println(tag + " : " + message);
     }
-    public static void d(final String tag, final String message, final Throwable e) {
+    public static void d(final String tag, final String message, final Throwable t) {
-        System.out.println(tag + " : " + message + " : " + e);
+        System.out.println(tag + " : " + message + " : " + t);
     }
     public static void e(final String tag, final String message) {
         d(tag, message);
     }
-    public static void e(final String tag, final String message, final Throwable e) {
+    public static void e(final String tag, final String message, final Throwable t) {
-        d(tag, message, e);
+        d(tag, message, t);
+    }
+    public static void w(final String tag, final String message) {
+        d(tag, message);
+    }
+    public static void w(final String tag, final String message, final Throwable t) {
+        d(tag, message, t);
     }
 }

tools/dicttool/compat/android/view/inputmethod/CompletionInfo.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.view.inputmethod;
+
+public class CompletionInfo {
+    public final String getText() { return ""; }
+}

tools/dicttool/compat/android/view/inputmethod/EditorInfo.java

@@ -0,0 +1,20 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.view.inputmethod;
+
+public class EditorInfo {
+}

tools/dicttool/compat/com/android/inputmethod/keyboard/Key.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.inputmethod.keyboard;
+
+public class Key {
+    public final int getX() { return 0; }
+    public final int getY() { return 0; }
+    public final int getWidth() { return 0; }
+    public final int getHeight() { return 0; }
+}

tools/dicttool/compat/com/android/inputmethod/keyboard/Keyboard.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.inputmethod.keyboard;
+
+public class Keyboard {
+    private final Key KEY = new Key();
+    public final Key getKey(final int i) { return KEY; }
+}

tools/dicttool/compat/com/android/inputmethod/keyboard/ProximityInfo.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.inputmethod.keyboard;
+
+public class ProximityInfo {
+    public long getNativeProximityInfo() { return 0l; }
+    private static native long setProximityInfoNative(String locale,
+            int displayWidth, int displayHeight, int gridWidth, int gridHeight,
+            int mostCommonKeyWidth, int mostCommonKeyHeight, int[] proximityCharsArray,
+            int keyCount, int[] keyXCoordinates, int[] keyYCoordinates, int[] keyWidths,
+            int[] keyHeights, int[] keyCharCodes, float[] sweetSpotCenterXs,
+            float[] sweetSpotCenterYs, float[] sweetSpotRadii);
+    private static native void releaseProximityInfoNative(long nativeProximityInfo);
+}

tools/dicttool/compat/com/android/inputmethod/latin/LatinIME.java

@@ -0,0 +1,20 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.inputmethod.latin;
+
+public class LatinIME {
+}

tools/dicttool/compat/com/android/inputmethod/latin/settings/AdditionalFeaturesSettingUtils.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.inputmethod.latin.settings;
+
+public class AdditionalFeaturesSettingUtils {
+    public static final int ADDITIONAL_FEATURES_SETTINGS_SIZE = 0;
+}

tools/dicttool/compat/com/android/inputmethod/latin/settings/SettingsValues.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.inputmethod.latin.settings;
+
+public class SettingsValues {
+    public boolean isWordCodePoint(final int code) {
+        return Character.isLetter(code);
+    }
+}

tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java

@@ -23,7 +23,7 @@ import com.android.inputmethod.latin.makedict.FormatSpec;
 import com.android.inputmethod.latin.makedict.FusionDictionary;
 import com.android.inputmethod.latin.makedict.MakedictLog;
 import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
-import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
+import com.android.inputmethod.latin.makedict.Ver2DictEncoder;
 import com.android.inputmethod.latin.makedict.Ver4DictEncoder;
 
 import java.io.BufferedWriter;
@@ -361,7 +361,7 @@ public class DictionaryMaker {
         if (version == FormatSpec.VERSION4) {
             dictEncoder = new Ver4DictEncoder(outputFile);
         } else {
-            dictEncoder = new Ver3DictEncoder(outputFile);
+            dictEncoder = new Ver2DictEncoder(outputFile);
         }
         dictEncoder.writeDictionary(dict, formatOptions);
     }

tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java

@@ -24,7 +24,7 @@ import com.android.inputmethod.latin.makedict.FusionDictionary;
 import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
 import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
 import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
-import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
+import com.android.inputmethod.latin.makedict.Ver2DictEncoder;
 
 import junit.framework.TestCase;
 
@@ -58,7 +58,7 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase {
                 Compress.getCompressedStream(
                         Compress.getCompressedStream(
                                 new BufferedOutputStream(new FileOutputStream(dst)))));
-        final DictEncoder dictEncoder = new Ver3DictEncoder(out);
+        final DictEncoder dictEncoder = new Ver2DictEncoder(out);
         dictEncoder.writeDictionary(dict, new FormatOptions(2, false));
 
         // Test for an actually compressed dictionary and its contents