Merge "Revert "ResearchLogging capture full n-gram data"" into jb-mr1-dev

java/res/values/strings.xml

@@ -261,8 +261,7 @@
     <string name="research_feedback_dialog_title" translatable="false">Send feedback</string>
     <!-- Text for checkbox option to include user data in feedback for research purposes [CHAR LIMIT=50] -->
     <!-- TODO: remove translatable=false attribute once text is stable -->
-    <!-- TODO: handle multilingual plurals -->
+    <string name="research_feedback_include_history_label" translatable="false">Include last 5 words entered</string>
-    <string name="research_feedback_include_history_label" translatable="false">Include last <xliff:g id="word">%d</xliff:g> words entered</string>
     <!-- Hint to user about the text entry field where they should enter research feedback [CHAR LIMIT=40] -->
     <!-- TODO: remove translatable=false attribute once text is stable -->
     <string name="research_feedback_hint" translatable="false">Enter your feedback here.</string>

java/src/com/android/inputmethod/latin/LatinIME.java

@@ -1247,6 +1247,11 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen
         }
         mLastKeyTime = when;
         mConnection.beginBatchEdit();
+
+        if (ProductionFlag.IS_EXPERIMENTAL) {
+            ResearchLogger.latinIME_onCodeInput(primaryCode, x, y);
+        }
+
         final KeyboardSwitcher switcher = mKeyboardSwitcher;
         // The space state depends only on the last character pressed and its own previous
         // state. Here, we revert the space state to neutral if the key is actually modifying
@@ -1328,9 +1333,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen
             mLastComposedWord.deactivate();
         mEnteredText = null;
         mConnection.endBatchEdit();
-        if (ProductionFlag.IS_EXPERIMENTAL) {
-            ResearchLogger.latinIME_onCodeInput(primaryCode, x, y);
-        }
     }
 
     // Called from PointerTracker through the KeyboardActionListener interface

java/src/com/android/inputmethod/research/FeedbackActivity.java

@@ -18,7 +18,10 @@ package com.android.inputmethod.research;
 
 import android.app.Activity;
 import android.os.Bundle;
+import android.text.Editable;
+import android.view.View;
 import android.widget.CheckBox;
+import android.widget.EditText;
 
 import com.android.inputmethod.latin.R;
 
@@ -28,11 +31,6 @@ public class FeedbackActivity extends Activity {
         super.onCreate(savedInstanceState);
         setContentView(R.layout.research_feedback_activity);
         final FeedbackLayout layout = (FeedbackLayout) findViewById(R.id.research_feedback_layout);
-        final CheckBox checkbox = (CheckBox) findViewById(R.id.research_feedback_include_history);
-        final CharSequence cs = checkbox.getText();
-        final String actualString = String.format(cs.toString(),
-                ResearchLogger.FEEDBACK_WORD_BUFFER_SIZE);
-        checkbox.setText(actualString);
         layout.setActivity(this);
     }
 

java/src/com/android/inputmethod/research/LogBuffer.java

@@ -1,111 +0,0 @@
-/*
- * Copyright (C) 2012 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License. You may obtain a copy of
- * the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package com.android.inputmethod.research;
-
-import java.util.LinkedList;
-
-/**
- * A buffer that holds a fixed number of LogUnits.
- *
- * LogUnits are added in and shifted out in temporal order.  Only a subset of the LogUnits are
- * actual words; the other LogUnits do not count toward the word limit.  Once the buffer reaches
- * capacity, adding another LogUnit that is a word evicts the oldest LogUnits out one at a time to
- * stay under the capacity limit.
- */
-public class LogBuffer {
-    protected final LinkedList<LogUnit> mLogUnits;
-    /* package for test */ int mWordCapacity;
-    // The number of members of mLogUnits that are actual words.
-    protected int mNumActualWords;
-
-    /**
-     * Create a new LogBuffer that can hold a fixed number of LogUnits that are words (and
-     * unlimited number of non-word LogUnits), and that outputs its result to a researchLog.
-     *
-     * @param wordCapacity maximum number of words
-     */
-    LogBuffer(final int wordCapacity) {
-        if (wordCapacity <= 0) {
-            throw new IllegalArgumentException("wordCapacity must be 1 or greater.");
-        }
-        mLogUnits = new LinkedList<LogUnit>();
-        mWordCapacity = wordCapacity;
-        mNumActualWords = 0;
-    }
-
-    /**
-     * Adds a new LogUnit to the front of the LIFO queue, evicting existing LogUnit's
-     * (oldest first) if word capacity is reached.
-     */
-    public void shiftIn(LogUnit newLogUnit) {
-        if (newLogUnit.getWord() == null) {
-            // This LogUnit isn't a word, so it doesn't count toward the word-limit.
-            mLogUnits.add(newLogUnit);
-            return;
-        }
-        if (mNumActualWords == mWordCapacity) {
-            shiftOutThroughFirstWord();
-        }
-        mLogUnits.add(newLogUnit);
-        mNumActualWords++; // Must be a word, or we wouldn't be here.
-    }
-
-    private void shiftOutThroughFirstWord() {
-        while (!mLogUnits.isEmpty()) {
-            final LogUnit logUnit = mLogUnits.removeFirst();
-            onShiftOut(logUnit);
-            if (logUnit.hasWord()) {
-                // Successfully shifted out a word-containing LogUnit and made space for the new
-                // LogUnit.
-                mNumActualWords--;
-                break;
-            }
-        }
-    }
-
-    /**
-     * Removes all LogUnits from the buffer without calling onShiftOut().
-     */
-    public void clear() {
-        mLogUnits.clear();
-        mNumActualWords = 0;
-    }
-
-    /**
-     * Called when a LogUnit is removed from the LogBuffer as a result of a shiftIn.  LogUnits are
-     * removed in the order entered.  This method is not called when shiftOut is called directly.
-     *
-     * Base class does nothing; subclasses may override.
-     */
-    protected void onShiftOut(LogUnit logUnit) {
-    }
-
-    /**
-     * Called to deliberately remove the oldest LogUnit.  Usually called when draining the
-     * LogBuffer.
-     */
-    public LogUnit shiftOut() {
-        if (mLogUnits.isEmpty()) {
-            return null;
-        }
-        final LogUnit logUnit = mLogUnits.removeFirst();
-        if (logUnit.hasWord()) {
-            mNumActualWords--;
-        }
-        return logUnit;
-    }
-}

java/src/com/android/inputmethod/research/LogUnit.java

@@ -1,81 +0,0 @@
-/*
- * Copyright (C) 2012 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License. You may obtain a copy of
- * the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package com.android.inputmethod.research;
-
-import java.util.ArrayList;
-
-/**
- * A group of log statements related to each other.
- *
- * A LogUnit is collection of LogStatements, each of which is generated by at a particular point
- * in the code.  (There is no LogStatement class; the data is stored across the instance variables
- * here.)  A single LogUnit's statements can correspond to all the calls made while in the same
- * composing region, or all the calls between committing the last composing region, and the first
- * character of the next composing region.
- *
- * Individual statements in a log may be marked as potentially private.  If so, then they are only
- * published to a ResearchLog if the ResearchLogger determines that publishing the entire LogUnit
- * will not violate the user's privacy.  Checks for this may include whether other LogUnits have
- * been published recently, or whether the LogUnit contains numbers, etc.
- */
-/* package */ class LogUnit {
-    private final ArrayList<String[]> mKeysList = new ArrayList<String[]>();
-    private final ArrayList<Object[]> mValuesList = new ArrayList<Object[]>();
-    private final ArrayList<Boolean> mIsPotentiallyPrivate = new ArrayList<Boolean>();
-    private String mWord;
-    private boolean mContainsDigit;
-
-    public void addLogStatement(final String[] keys, final Object[] values,
-            final Boolean isPotentiallyPrivate) {
-        mKeysList.add(keys);
-        mValuesList.add(values);
-        mIsPotentiallyPrivate.add(isPotentiallyPrivate);
-    }
-
-    public void publishTo(final ResearchLog researchLog, final boolean isIncludingPrivateData) {
-        final int size = mKeysList.size();
-        for (int i = 0; i < size; i++) {
-            if (!mIsPotentiallyPrivate.get(i) || isIncludingPrivateData) {
-                researchLog.outputEvent(mKeysList.get(i), mValuesList.get(i));
-            }
-        }
-    }
-
-    public void setWord(String word) {
-        mWord = word;
-    }
-
-    public String getWord() {
-        return mWord;
-    }
-
-    public boolean hasWord() {
-        return mWord != null;
-    }
-
-    public void setContainsDigit() {
-        mContainsDigit = true;
-    }
-
-    public boolean hasDigit() {
-        return mContainsDigit;
-    }
-
-    public boolean isEmpty() {
-        return mKeysList.isEmpty();
-    }
-}

java/src/com/android/inputmethod/research/MainLogBuffer.java

@@ -1,127 +0,0 @@
-/*
- * Copyright (C) 2012 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License. You may obtain a copy of
- * the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package com.android.inputmethod.research;
-
-import com.android.inputmethod.latin.Dictionary;
-import com.android.inputmethod.latin.Suggest;
-
-import java.util.Random;
-
-public class MainLogBuffer extends LogBuffer {
-    // The size of the n-grams logged.  E.g. N_GRAM_SIZE = 2 means to sample bigrams.
-    private static final int N_GRAM_SIZE = 2;
-    // The number of words between n-grams to omit from the log.
-    private static final int DEFAULT_NUMBER_OF_WORDS_BETWEEN_SAMPLES = 18;
-
-    private final ResearchLog mResearchLog;
-    private Suggest mSuggest;
-
-    // The minimum periodicity with which n-grams can be sampled.  E.g. mWinWordPeriod is 10 if
-    // every 10th bigram is sampled, i.e., words 1-8 are not, but the bigram at words 9 and 10, etc.
-    // for 11-18, and the bigram at words 19 and 20.  If an n-gram is not safe (e.g. it  contains a
-    // number in the middle or an out-of-vocabulary word), then sampling is delayed until a safe
-    // n-gram does appear.
-    /* package for test */ int mMinWordPeriod;
-
-    // Counter for words left to suppress before an n-gram can be sampled.  Reset to mMinWordPeriod
-    // after a sample is taken.
-    /* package for test */ int mWordsUntilSafeToSample;
-
-    public MainLogBuffer(final ResearchLog researchLog) {
-        super(N_GRAM_SIZE);
-        mResearchLog = researchLog;
-        mMinWordPeriod = DEFAULT_NUMBER_OF_WORDS_BETWEEN_SAMPLES + N_GRAM_SIZE;
-        final Random random = new Random();
-        mWordsUntilSafeToSample = random.nextInt(mMinWordPeriod);
-    }
-
-    public void setSuggest(Suggest suggest) {
-        mSuggest = suggest;
-    }
-
-    @Override
-    public void shiftIn(final LogUnit newLogUnit) {
-        super.shiftIn(newLogUnit);
-        if (newLogUnit.hasWord()) {
-            if (mWordsUntilSafeToSample > 0) {
-                mWordsUntilSafeToSample--;
-            }
-        }
-    }
-
-    public void resetWordCounter() {
-        mWordsUntilSafeToSample = mMinWordPeriod;
-    }
-
-    /**
-     * Determines whether the content of the MainLogBuffer can be safely uploaded in its complete
-     * form and still protect the user's privacy.
-     *
-     * The size of the MainLogBuffer is just enough to hold one n-gram, its corrections, and any
-     * non-character data that is typed between words.  The decision about privacy is made based on
-     * the buffer's entire content.  If it is decided that the privacy risks are too great to upload
-     * the contents of this buffer, a censored version of the LogItems may still be uploaded.  E.g.,
-     * the screen orientation and other characteristics about the device can be uploaded without
-     * revealing much about the user.
-     */
-    public boolean isSafeToLog() {
-        // Check that we are not sampling too frequently.  Having sampled recently might disclose
-        // too much of the user's intended meaning.
-        if (mWordsUntilSafeToSample > 0) {
-            return false;
-        }
-        if (mSuggest == null || !mSuggest.hasMainDictionary()) {
-            // Main dictionary is unavailable.  Since we cannot check it, we cannot tell if a word
-            // is out-of-vocabulary or not.  Therefore, we must judge the entire buffer contents to
-            // potentially pose a privacy risk.
-            return false;
-        }
-        // Reload the dictionary in case it has changed (e.g., because the user has changed
-        // languages).
-        final Dictionary dictionary = mSuggest.getMainDictionary();
-        if (dictionary == null) {
-            return false;
-        }
-        // Check each word in the buffer.  If any word poses a privacy threat, we cannot upload the
-        // complete buffer contents in detail.
-        final int length = mLogUnits.size();
-        for (int i = 0; i < length; i++) {
-            final LogUnit logUnit = mLogUnits.get(i);
-            final String word = logUnit.getWord();
-            if (word == null) {
-                // Digits outside words are a privacy threat.
-                if (logUnit.hasDigit()) {
-                    return false;
-                }
-            } else {
-                // Words not in the dictionary are a privacy threat.
-                if (!(dictionary.isValidWord(word))) {
-                    return false;
-                }
-            }
-        }
-        // All checks have passed; this buffer's content can be safely uploaded.
-        return true;
-    }
-
-    @Override
-    protected void onShiftOut(LogUnit logUnit) {
-        if (mResearchLog != null) {
-            mResearchLog.publish(logUnit, false /* isIncludingPrivateData */);
-        }
-    }
-}

java/src/com/android/inputmethod/research/ResearchLog.java

@@ -26,6 +26,7 @@ import com.android.inputmethod.keyboard.Key;
 import com.android.inputmethod.latin.SuggestedWords;
 import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
 import com.android.inputmethod.latin.define.ProductionFlag;
+import com.android.inputmethod.research.ResearchLogger.LogUnit;
 
 import java.io.BufferedWriter;
 import java.io.File;
@@ -36,7 +37,6 @@ import java.io.OutputStreamWriter;
 import java.util.Map;
 import java.util.concurrent.Callable;
 import java.util.concurrent.Executors;
-import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
@@ -51,22 +51,21 @@ import java.util.concurrent.TimeUnit;
  */
 public class ResearchLog {
     private static final String TAG = ResearchLog.class.getSimpleName();
-    private static final boolean DEBUG = false;
-    private static final long FLUSH_DELAY_IN_MS = 1000 * 5;
-    private static final int ABORT_TIMEOUT_IN_MS = 1000 * 4;
-
-    /* package */ final ScheduledExecutorService mExecutor;
-    /* package */ final File mFile;
-    private JsonWriter mJsonWriter = NULL_JSON_WRITER;
-    // true if at least one byte of data has been written out to the log file.  This must be
-    // remembered because JsonWriter requires that calls matching calls to beginObject and
-    // endObject, as well as beginArray and endArray, and the file is opened lazily, only when
-    // it is certain that data will be written.  Alternatively, the matching call exceptions
-    // could be caught, but this might suppress other errors.
-    private boolean mHasWrittenData = false;
-
     private static final JsonWriter NULL_JSON_WRITER = new JsonWriter(
             new OutputStreamWriter(new NullOutputStream()));
+
+    final ScheduledExecutorService mExecutor;
+    /* package */ final File mFile;
+    private JsonWriter mJsonWriter = NULL_JSON_WRITER;
+
+    private int mLoggingState;
+    private static final int LOGGING_STATE_UNSTARTED = 0;
+    private static final int LOGGING_STATE_READY = 1;   // don't create file until necessary
+    private static final int LOGGING_STATE_RUNNING = 2;
+    private static final int LOGGING_STATE_STOPPING = 3;
+    private static final int LOGGING_STATE_STOPPED = 4;
+    private static final long FLUSH_DELAY_IN_MS = 1000 * 5;
+
     private static class NullOutputStream extends OutputStream {
         /** {@inheritDoc} */
         @Override
@@ -85,81 +84,128 @@ public class ResearchLog {
         }
     }
 
-    public ResearchLog(final File outputFile) {
+    public ResearchLog(File outputFile) {
+        mExecutor = Executors.newSingleThreadScheduledExecutor();
         if (outputFile == null) {
             throw new IllegalArgumentException();
         }
-        mExecutor = Executors.newSingleThreadScheduledExecutor();
         mFile = outputFile;
+        mLoggingState = LOGGING_STATE_UNSTARTED;
     }
 
-    public synchronized void close() {
+    public synchronized void start() throws IOException {
-        mExecutor.submit(new Callable<Object>() {
+        switch (mLoggingState) {
-            @Override
+            case LOGGING_STATE_UNSTARTED:
-            public Object call() throws Exception {
+                mLoggingState = LOGGING_STATE_READY;
-                try {
+                break;
-                    if (mHasWrittenData) {
+            case LOGGING_STATE_READY:
-                        mJsonWriter.endArray();
+            case LOGGING_STATE_RUNNING:
-                        mJsonWriter.flush();
+            case LOGGING_STATE_STOPPING:
-                        mJsonWriter.close();
+            case LOGGING_STATE_STOPPED:
-                        mHasWrittenData = false;
+                break;
+        }
+    }
+
+    public synchronized void stop() {
+        switch (mLoggingState) {
+            case LOGGING_STATE_UNSTARTED:
+                mLoggingState = LOGGING_STATE_STOPPED;
+                break;
+            case LOGGING_STATE_READY:
+            case LOGGING_STATE_RUNNING:
+                mExecutor.submit(new Callable<Object>() {
+                    @Override
+                    public Object call() throws Exception {
+                        try {
+                            mJsonWriter.endArray();
+                            mJsonWriter.flush();
+                            mJsonWriter.close();
+                        } finally {
+                            boolean success = mFile.setWritable(false, false);
+                            mLoggingState = LOGGING_STATE_STOPPED;
+                        }
+                        return null;
                     }
-                } catch (Exception e) {
+                });
-                    Log.d(TAG, "error when closing ResearchLog:");
+                removeAnyScheduledFlush();
-                    e.printStackTrace();
+                mExecutor.shutdown();
-                } finally {
+                mLoggingState = LOGGING_STATE_STOPPING;
-                    if (mFile.exists()) {
+                break;
-                        mFile.setWritable(false, false);
+            case LOGGING_STATE_STOPPING:
-                    }
+            case LOGGING_STATE_STOPPED:
-                }
+        }
-                return null;
+    }
-            }
+
-        });
+    public boolean isAlive() {
+        switch (mLoggingState) {
+            case LOGGING_STATE_UNSTARTED:
+            case LOGGING_STATE_READY:
+            case LOGGING_STATE_RUNNING:
+                return true;
+        }
+        return false;
+    }
+
+    public void waitUntilStopped(final int timeoutInMs) throws InterruptedException {
         removeAnyScheduledFlush();
         mExecutor.shutdown();
+        mExecutor.awaitTermination(timeoutInMs, TimeUnit.MILLISECONDS);
     }
 
-    private boolean mIsAbortSuccessful;
-
     public synchronized void abort() {
-        mExecutor.submit(new Callable<Object>() {
+        switch (mLoggingState) {
-            @Override
+            case LOGGING_STATE_UNSTARTED:
-            public Object call() throws Exception {
+                mLoggingState = LOGGING_STATE_STOPPED;
-                try {
+                isAbortSuccessful = true;
-                    if (mHasWrittenData) {
+                break;
-                        mJsonWriter.endArray();
+            case LOGGING_STATE_READY:
-                        mJsonWriter.close();
+            case LOGGING_STATE_RUNNING:
-                        mHasWrittenData = false;
+                mExecutor.submit(new Callable<Object>() {
+                    @Override
+                    public Object call() throws Exception {
+                        try {
+                            mJsonWriter.endArray();
+                            mJsonWriter.close();
+                        } finally {
+                            isAbortSuccessful = mFile.delete();
+                        }
+                        return null;
                     }
-                } finally {
+                });
-                    mIsAbortSuccessful = mFile.delete();
+                removeAnyScheduledFlush();
-                }
+                mExecutor.shutdown();
-                return null;
+                mLoggingState = LOGGING_STATE_STOPPING;
-            }
+                break;
-        });
+            case LOGGING_STATE_STOPPING:
-        removeAnyScheduledFlush();
+            case LOGGING_STATE_STOPPED:
-        mExecutor.shutdown();
+        }
     }
 
-    public boolean blockingAbort() throws InterruptedException {
+    private boolean isAbortSuccessful;
-        abort();
+    public boolean isAbortSuccessful() {
-        mExecutor.awaitTermination(ABORT_TIMEOUT_IN_MS, TimeUnit.MILLISECONDS);
+        return isAbortSuccessful;
-        return mIsAbortSuccessful;
-    }
-
-    public void awaitTermination(int delay, TimeUnit timeUnit) throws InterruptedException {
-        mExecutor.awaitTermination(delay, timeUnit);
     }
 
     /* package */ synchronized void flush() {
-        removeAnyScheduledFlush();
+        switch (mLoggingState) {
-        mExecutor.submit(mFlushCallable);
+            case LOGGING_STATE_UNSTARTED:
+                break;
+            case LOGGING_STATE_READY:
+            case LOGGING_STATE_RUNNING:
+                removeAnyScheduledFlush();
+                mExecutor.submit(mFlushCallable);
+                break;
+            case LOGGING_STATE_STOPPING:
+            case LOGGING_STATE_STOPPED:
+        }
     }
 
-    private final Callable<Object> mFlushCallable = new Callable<Object>() {
+    private Callable<Object> mFlushCallable = new Callable<Object>() {
         @Override
         public Object call() throws Exception {
-            mJsonWriter.flush();
+            if (mLoggingState == LOGGING_STATE_RUNNING) {
+                mJsonWriter.flush();
+            }
             return null;
         }
     };
@@ -178,40 +224,56 @@ public class ResearchLog {
         mFlushFuture = mExecutor.schedule(mFlushCallable, FLUSH_DELAY_IN_MS, TimeUnit.MILLISECONDS);
     }
 
-    public synchronized void publish(final LogUnit logUnit, final boolean isIncludingPrivateData) {
+    public synchronized void publishPublicEvents(final LogUnit logUnit) {
-        try {
+        switch (mLoggingState) {
-            mExecutor.submit(new Callable<Object>() {
+            case LOGGING_STATE_UNSTARTED:
-                @Override
+                break;
-                public Object call() throws Exception {
+            case LOGGING_STATE_READY:
-                    logUnit.publishTo(ResearchLog.this, isIncludingPrivateData);
+            case LOGGING_STATE_RUNNING:
-                    scheduleFlush();
+                mExecutor.submit(new Callable<Object>() {
-                    return null;
+                    @Override
-                }
+                    public Object call() throws Exception {
-            });
+                        logUnit.publishPublicEventsTo(ResearchLog.this);
-        } catch (RejectedExecutionException e) {
+                        scheduleFlush();
-            // TODO: Add code to record loss of data, and report.
+                        return null;
+                    }
+                });
+                break;
+            case LOGGING_STATE_STOPPING:
+            case LOGGING_STATE_STOPPED:
+        }
+    }
+
+    public synchronized void publishAllEvents(final LogUnit logUnit) {
+        switch (mLoggingState) {
+            case LOGGING_STATE_UNSTARTED:
+                break;
+            case LOGGING_STATE_READY:
+            case LOGGING_STATE_RUNNING:
+                mExecutor.submit(new Callable<Object>() {
+                    @Override
+                    public Object call() throws Exception {
+                        logUnit.publishAllEventsTo(ResearchLog.this);
+                        scheduleFlush();
+                        return null;
+                    }
+                });
+                break;
+            case LOGGING_STATE_STOPPING:
+            case LOGGING_STATE_STOPPED:
         }
     }
 
     private static final String CURRENT_TIME_KEY = "_ct";
     private static final String UPTIME_KEY = "_ut";
     private static final String EVENT_TYPE_KEY = "_ty";
-
     void outputEvent(final String[] keys, final Object[] values) {
-        // Not thread safe.
+        // not thread safe.
-        if (keys.length == 0) {
-            return;
-        }
-        if (DEBUG) {
-            if (keys.length != values.length + 1) {
-                Log.d(TAG, "Key and Value list sizes do not match. " + keys[0]);
-            }
-        }
         try {
             if (mJsonWriter == NULL_JSON_WRITER) {
                 mJsonWriter = new JsonWriter(new BufferedWriter(new FileWriter(mFile)));
+                mJsonWriter.setLenient(true);
                 mJsonWriter.beginArray();
-                mHasWrittenData = true;
             }
             mJsonWriter.beginObject();
             mJsonWriter.name(CURRENT_TIME_KEY).value(System.currentTimeMillis());
@@ -221,8 +283,8 @@ public class ResearchLog {
             for (int i = 0; i < length; i++) {
                 mJsonWriter.name(keys[i + 1]);
                 Object value = values[i];
-                if (value instanceof CharSequence) {
+                if (value instanceof String) {
-                    mJsonWriter.value(value.toString());
+                    mJsonWriter.value((String) value);
                 } else if (value instanceof Number) {
                     mJsonWriter.value((Number) value);
                 } else if (value instanceof Boolean) {
@@ -269,11 +331,14 @@ public class ResearchLog {
                     SuggestedWords words = (SuggestedWords) value;
                     mJsonWriter.beginObject();
                     mJsonWriter.name("typedWordValid").value(words.mTypedWordValid);
-                    mJsonWriter.name("willAutoCorrect").value(words.mWillAutoCorrect);
+                    mJsonWriter.name("willAutoCorrect")
+                        .value(words.mWillAutoCorrect);
                     mJsonWriter.name("isPunctuationSuggestions")
-                            .value(words.mIsPunctuationSuggestions);
+                        .value(words.mIsPunctuationSuggestions);
-                    mJsonWriter.name("isObsoleteSuggestions").value(words.mIsObsoleteSuggestions);
+                    mJsonWriter.name("isObsoleteSuggestions")
-                    mJsonWriter.name("isPrediction").value(words.mIsPrediction);
+                        .value(words.mIsObsoleteSuggestions);
+                    mJsonWriter.name("isPrediction")
+                        .value(words.mIsPrediction);
                     mJsonWriter.name("words");
                     mJsonWriter.beginArray();
                     final int size = words.size();
@@ -298,8 +363,8 @@ public class ResearchLog {
             try {
                 mJsonWriter.close();
             } catch (IllegalStateException e1) {
-                // Assume that this is just the json not being terminated properly.
+                // assume that this is just the json not being terminated properly.
-                // Ignore
+                // ignore
             } catch (IOException e1) {
                 e1.printStackTrace();
             } finally {

java/src/com/android/inputmethod/research/ResearchLogUploader.java

@@ -27,6 +27,7 @@ import android.os.BatteryManager;
 import android.util.Log;
 
 import com.android.inputmethod.latin.R;
+import com.android.inputmethod.latin.R.string;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -47,7 +48,6 @@ public final class ResearchLogUploader {
     private static final String TAG = ResearchLogUploader.class.getSimpleName();
     private static final int UPLOAD_INTERVAL_IN_MS = 1000 * 60 * 15; // every 15 min
     private static final int BUF_SIZE = 1024 * 8;
-    protected static final int TIMEOUT_IN_MS = 1000 * 4;
 
     private final boolean mCanUpload;
     private final Context mContext;
@@ -55,6 +55,8 @@ public final class ResearchLogUploader {
     private final URL mUrl;
     private final ScheduledExecutorService mExecutor;
 
+    private Runnable doUploadRunnable = new UploadRunnable(null, false);
+
     public ResearchLogUploader(final Context context, final File filesDir) {
         mContext = context;
         mFilesDir = filesDir;
@@ -91,15 +93,11 @@ public final class ResearchLogUploader {
 
     public void start() {
         if (mCanUpload) {
-            mExecutor.scheduleWithFixedDelay(new UploadRunnable(null /* logToWaitFor */,
+            Log.d(TAG, "scheduling regular uploading");
-                    null /* callback */, false /* forceUpload */),
+            mExecutor.scheduleWithFixedDelay(doUploadRunnable, UPLOAD_INTERVAL_IN_MS,
-                    UPLOAD_INTERVAL_IN_MS, UPLOAD_INTERVAL_IN_MS, TimeUnit.MILLISECONDS);
+                    UPLOAD_INTERVAL_IN_MS, TimeUnit.MILLISECONDS);
-        }
+        } else {
-    }
+            Log.d(TAG, "no permission to upload");
-
-    public void uploadAfterCompletion(final ResearchLog researchLog, final Callback callback) {
-        if (mCanUpload) {
-            mExecutor.submit(new UploadRunnable(researchLog, callback, true /* forceUpload */));
         }
     }
 
@@ -108,8 +106,7 @@ public final class ResearchLogUploader {
         // another upload happening right now, as it may have missed the latest changes.
         // TODO: Reschedule regular upload tests starting from now.
         if (mCanUpload) {
-            mExecutor.submit(new UploadRunnable(null /* logToWaitFor */, callback,
+            mExecutor.submit(new UploadRunnable(callback, true));
-                    true /* forceUpload */));
         }
     }
 
@@ -133,33 +130,19 @@ public final class ResearchLogUploader {
     }
 
     class UploadRunnable implements Runnable {
-        private final ResearchLog mLogToWaitFor;
         private final Callback mCallback;
         private final boolean mForceUpload;
 
-        public UploadRunnable(final ResearchLog logToWaitFor, final Callback callback,
+        public UploadRunnable(final Callback callback, final boolean forceUpload) {
-                final boolean forceUpload) {
-            mLogToWaitFor = logToWaitFor;
             mCallback = callback;
             mForceUpload = forceUpload;
         }
 
         @Override
         public void run() {
-            if (mLogToWaitFor != null) {
-                waitFor(mLogToWaitFor);
-            }
             doUpload();
         }
 
-        private void waitFor(final ResearchLog researchLog) {
-            try {
-                researchLog.awaitTermination(TIMEOUT_IN_MS, TimeUnit.MILLISECONDS);
-            } catch (InterruptedException e) {
-                e.printStackTrace();
-            }
-        }
-
         private void doUpload() {
             if (!mForceUpload && (!isExternallyPowered() || !hasWifiConnection())) {
                 return;

java/src/com/android/inputmethod/research/ResearchLogger.java

@@ -67,8 +67,11 @@ import com.android.inputmethod.latin.SuggestedWords;
 import com.android.inputmethod.latin.define.ProductionFlag;
 
 import java.io.File;
+import java.io.IOException;
 import java.text.SimpleDateFormat;
+import java.util.ArrayList;
 import java.util.Date;
+import java.util.List;
 import java.util.Locale;
 import java.util.UUID;
 
@@ -94,21 +97,24 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
             new SimpleDateFormat("yyyyMMddHHmmssS", Locale.US);
     private static final boolean IS_SHOWING_INDICATOR = true;
     private static final boolean IS_SHOWING_INDICATOR_CLEARLY = false;
-    public static final int FEEDBACK_WORD_BUFFER_SIZE = 5;
 
     // constants related to specific log points
     private static final String WHITESPACE_SEPARATORS = " \t\n\r";
     private static final int MAX_INPUTVIEW_LENGTH_TO_CAPTURE = 8192; // must be >=1
     private static final String PREF_RESEARCH_LOGGER_UUID_STRING = "pref_research_logger_uuid";
+    private static final int ABORT_TIMEOUT_IN_MS = 10 * 1000; // timeout to notify user
 
     private static final ResearchLogger sInstance = new ResearchLogger();
     // to write to a different filename, e.g., for testing, set mFile before calling start()
     /* package */ File mFilesDir;
     /* package */ String mUUIDString;
     /* package */ ResearchLog mMainResearchLog;
-    /* package */ ResearchLog mFeedbackLog;
+    // The mIntentionalResearchLog records all events for the session, private or not (excepting
-    /* package */ MainLogBuffer mMainLogBuffer;
+    // passwords).  It is written to permanent storage only if the user explicitly commands
-    /* package */ LogBuffer mFeedbackLogBuffer;
+    // the system to do so.
+    /* package */ ResearchLog mIntentionalResearchLog;
+    // LogUnits are queued here and released only when the user requests the intentional log.
+    private List<LogUnit> mIntentionalResearchLogQueue = new ArrayList<LogUnit>();
 
     private boolean mIsPasswordView = false;
     private boolean mIsLoggingSuspended = false;
@@ -133,9 +139,8 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
     private KeyboardSwitcher mKeyboardSwitcher;
     private InputMethodService mInputMethodService;
     private final Statistics mStatistics;
-    private ResearchLogUploader mResearchLogUploader;
 
-    private LogUnit mCurrentLogUnit = new LogUnit();
+    private ResearchLogUploader mResearchLogUploader;
 
     private ResearchLogger() {
         mStatistics = Statistics.getInstance();
@@ -259,16 +264,6 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
         e.apply();
     }
 
-    private void setLoggingAllowed(boolean enableLogging) {
-        if (mPrefs == null) {
-            return;
-        }
-        Editor e = mPrefs.edit();
-        e.putBoolean(PREF_USABILITY_STUDY_MODE, enableLogging);
-        e.apply();
-        sIsLogging = enableLogging;
-    }
-
     private File createLogFile(File filesDir) {
         final StringBuilder sb = new StringBuilder();
         sb.append(FILENAME_PREFIX).append('-');
@@ -315,58 +310,97 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
             Log.w(TAG, "IME storage directory does not exist.  Cannot start logging.");
             return;
         }
-        if (mMainLogBuffer == null) {
+        try {
-            mMainResearchLog = new ResearchLog(createLogFile(mFilesDir));
+            if (mMainResearchLog == null || !mMainResearchLog.isAlive()) {
-            mMainLogBuffer = new MainLogBuffer(mMainResearchLog);
+                mMainResearchLog = new ResearchLog(createLogFile(mFilesDir));
-            mMainLogBuffer.setSuggest(mSuggest);
+            }
-        }
+            mMainResearchLog.start();
-        if (mFeedbackLogBuffer == null) {
+            if (mIntentionalResearchLog == null || !mIntentionalResearchLog.isAlive()) {
-            mFeedbackLog = new ResearchLog(createLogFile(mFilesDir));
+                mIntentionalResearchLog = new ResearchLog(createLogFile(mFilesDir));
-            // LogBuffer is one more than FEEDBACK_WORD_BUFFER_SIZE, because it must also hold
+            }
-            // the feedback LogUnit itself.
+            mIntentionalResearchLog.start();
-            mFeedbackLogBuffer = new LogBuffer(FEEDBACK_WORD_BUFFER_SIZE + 1);
+        } catch (IOException e) {
+            Log.w(TAG, "Could not start ResearchLogger.");
         }
     }
 
     /* package */ void stop() {
         logStatistics();
-        commitCurrentLogUnit();
+        publishLogUnit(mCurrentLogUnit, true);
+        mCurrentLogUnit = new LogUnit();
 
-        if (mMainLogBuffer != null) {
+        if (mMainResearchLog != null) {
-            publishLogBuffer(mMainLogBuffer, mMainResearchLog, false /* isIncludingPrivateData */);
+            mMainResearchLog.stop();
-            mMainResearchLog.close();
-            mMainLogBuffer = null;
         }
-        if (mFeedbackLogBuffer != null) {
+        if (mIntentionalResearchLog != null) {
-            mFeedbackLog.close();
+            mIntentionalResearchLog.stop();
-            mFeedbackLogBuffer = null;
         }
     }
 
+    private static final String[] EVENTKEYS_STATISTICS = {
+        "Statistics", "charCount", "letterCount", "numberCount", "spaceCount", "deleteOpsCount",
+        "wordCount", "isEmptyUponStarting", "isEmptinessStateKnown", "averageTimeBetweenKeys",
+        "averageTimeBeforeDelete", "averageTimeDuringRepeatedDelete", "averageTimeAfterDelete"
+    };
+    private static void logStatistics() {
+        final ResearchLogger researchLogger = getInstance();
+        final Statistics statistics = researchLogger.mStatistics;
+        final Object[] values = {
+                statistics.mCharCount, statistics.mLetterCount, statistics.mNumberCount,
+                statistics.mSpaceCount, statistics.mDeleteKeyCount,
+                statistics.mWordCount, statistics.mIsEmptyUponStarting,
+                statistics.mIsEmptinessStateKnown, statistics.mKeyCounter.getAverageTime(),
+                statistics.mBeforeDeleteKeyCounter.getAverageTime(),
+                statistics.mDuringRepeatedDeleteKeysCounter.getAverageTime(),
+                statistics.mAfterDeleteKeyCounter.getAverageTime()
+        };
+        researchLogger.enqueueEvent(EVENTKEYS_STATISTICS, values);
+    }
+
+    private void setLoggingAllowed(boolean enableLogging) {
+        if (mPrefs == null) {
+            return;
+        }
+        Editor e = mPrefs.edit();
+        e.putBoolean(PREF_USABILITY_STUDY_MODE, enableLogging);
+        e.apply();
+        sIsLogging = enableLogging;
+    }
+
     public boolean abort() {
         boolean didAbortMainLog = false;
-        if (mMainLogBuffer != null) {
+        if (mMainResearchLog != null) {
-            mMainLogBuffer.clear();
+            mMainResearchLog.abort();
             try {
-                didAbortMainLog = mMainResearchLog.blockingAbort();
+                mMainResearchLog.waitUntilStopped(ABORT_TIMEOUT_IN_MS);
             } catch (InterruptedException e) {
-                // Don't know whether this succeeded or not.  We assume not; this is reported
+                // interrupted early.  carry on.
-                // to the caller.
             }
-            mMainLogBuffer = null;
+            if (mMainResearchLog.isAbortSuccessful()) {
+                didAbortMainLog = true;
+            }
+            mMainResearchLog = null;
         }
-        boolean didAbortFeedbackLog = false;
+        boolean didAbortIntentionalLog = false;
-        if (mFeedbackLogBuffer != null) {
+        if (mIntentionalResearchLog != null) {
-            mFeedbackLogBuffer.clear();
+            mIntentionalResearchLog.abort();
             try {
-                didAbortFeedbackLog = mFeedbackLog.blockingAbort();
+                mIntentionalResearchLog.waitUntilStopped(ABORT_TIMEOUT_IN_MS);
             } catch (InterruptedException e) {
-                // Don't know whether this succeeded or not.  We assume not; this is reported
+                // interrupted early.  carry on.
-                // to the caller.
             }
-            mFeedbackLogBuffer = null;
+            if (mIntentionalResearchLog.isAbortSuccessful()) {
+                didAbortIntentionalLog = true;
+            }
+            mIntentionalResearchLog = null;
+        }
+        return didAbortMainLog && didAbortIntentionalLog;
+    }
+
+    /* package */ void flush() {
+        if (mMainResearchLog != null) {
+            mMainResearchLog.flush();
         }
-        return didAbortMainLog && didAbortFeedbackLog;
     }
 
     private void restart() {
@@ -470,39 +504,79 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
         latinIME.launchKeyboardedDialogActivity(FeedbackActivity.class);
     }
 
-    private static final String[] EVENTKEYS_FEEDBACK = {
+    private ResearchLog mFeedbackLog;
-        "UserTimestamp", "contents"
+    private List<LogUnit> mFeedbackQueue;
-    };
+    private ResearchLog mSavedMainResearchLog;
+    private ResearchLog mSavedIntentionalResearchLog;
+    private List<LogUnit> mSavedIntentionalResearchLogQueue;
+
+    private void saveLogsForFeedback() {
+        mFeedbackLog = mIntentionalResearchLog;
+        if (mIntentionalResearchLogQueue != null) {
+            mFeedbackQueue = new ArrayList<LogUnit>(mIntentionalResearchLogQueue);
+        } else {
+            mFeedbackQueue = null;
+        }
+        mSavedMainResearchLog = mMainResearchLog;
+        mSavedIntentionalResearchLog = mIntentionalResearchLog;
+        mSavedIntentionalResearchLogQueue = mIntentionalResearchLogQueue;
+
+        mMainResearchLog = null;
+        mIntentionalResearchLog = null;
+        mIntentionalResearchLogQueue = new ArrayList<LogUnit>();
+    }
+
+    private static final int LOG_DRAIN_TIMEOUT_IN_MS = 1000 * 5;
     public void sendFeedback(final String feedbackContents, final boolean includeHistory) {
-        if (mFeedbackLogBuffer == null) {
+        if (includeHistory && mFeedbackLog != null) {
-            return;
+            try {
+                LogUnit headerLogUnit = new LogUnit();
+                headerLogUnit.addLogAtom(EVENTKEYS_INTENTIONAL_LOG, EVENTKEYS_NULLVALUES, false);
+                mFeedbackLog.publishAllEvents(headerLogUnit);
+                for (LogUnit logUnit : mFeedbackQueue) {
+                    mFeedbackLog.publishAllEvents(logUnit);
+                }
+                userFeedback(mFeedbackLog, feedbackContents);
+                mFeedbackLog.stop();
+                try {
+                    mFeedbackLog.waitUntilStopped(LOG_DRAIN_TIMEOUT_IN_MS);
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
+                mIntentionalResearchLog = new ResearchLog(createLogFile(mFilesDir));
+                mIntentionalResearchLog.start();
+            } catch (IOException e) {
+                e.printStackTrace();
+            } finally {
+                mIntentionalResearchLogQueue.clear();
+            }
+            mResearchLogUploader.uploadNow(null);
+        } else {
+            // create a separate ResearchLog just for feedback
+            final ResearchLog feedbackLog = new ResearchLog(createLogFile(mFilesDir));
+            try {
+                feedbackLog.start();
+                userFeedback(feedbackLog, feedbackContents);
+                feedbackLog.stop();
+                feedbackLog.waitUntilStopped(LOG_DRAIN_TIMEOUT_IN_MS);
+                mResearchLogUploader.uploadNow(null);
+            } catch (IOException e) {
+                e.printStackTrace();
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
         }
-        if (!includeHistory) {
-            mFeedbackLogBuffer.clear();
-        }
-        commitCurrentLogUnit();
-        final LogUnit feedbackLogUnit = new LogUnit();
-        final Object[] values = {
-            feedbackContents
-        };
-        feedbackLogUnit.addLogStatement(EVENTKEYS_FEEDBACK, values,
-                false /* isPotentiallyPrivate */);
-        mFeedbackLogBuffer.shiftIn(feedbackLogUnit);
-        publishLogBuffer(mFeedbackLogBuffer, mFeedbackLog, true /* isIncludingPrivateData */);
-        mFeedbackLog.close();
-        mResearchLogUploader.uploadAfterCompletion(mFeedbackLog, null);
-        mFeedbackLog = new ResearchLog(createLogFile(mFilesDir));
     }
 
     public void onLeavingSendFeedbackDialog() {
         mInFeedbackDialog = false;
+        mMainResearchLog = mSavedMainResearchLog;
+        mIntentionalResearchLog = mSavedIntentionalResearchLog;
+        mIntentionalResearchLogQueue = mSavedIntentionalResearchLogQueue;
     }
 
     public void initSuggest(Suggest suggest) {
         mSuggest = suggest;
-        if (mMainLogBuffer != null) {
-            mMainLogBuffer.setSuggest(mSuggest);
-        }
     }
 
     private void setIsPasswordView(boolean isPasswordView) {
@@ -510,7 +584,7 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
     }
 
     private boolean isAllowedToLog() {
-        return !mIsPasswordView && !mIsLoggingSuspended && sIsLogging && !mInFeedbackDialog;
+        return !mIsPasswordView && !mIsLoggingSuspended && sIsLogging;
     }
 
     public void requestIndicatorRedraw() {
@@ -557,8 +631,13 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
         }
     }
 
+    private static final String CURRENT_TIME_KEY = "_ct";
+    private static final String UPTIME_KEY = "_ut";
+    private static final String EVENT_TYPE_KEY = "_ty";
     private static final Object[] EVENTKEYS_NULLVALUES = {};
 
+    private LogUnit mCurrentLogUnit = new LogUnit();
+
     /**
      * Buffer a research log event, flagging it as privacy-sensitive.
      *
@@ -574,14 +653,10 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
             final Object[] values) {
         assert values.length + 1 == keys.length;
         if (isAllowedToLog()) {
-            mCurrentLogUnit.addLogStatement(keys, values, true /* isPotentiallyPrivate */);
+            mCurrentLogUnit.addLogAtom(keys, values, true);
         }
     }
 
-    private void setCurrentLogUnitContainsDigitFlag() {
-        mCurrentLogUnit.setContainsDigit();
-    }
-
     /**
      * Buffer a research log event, flaggint it as not privacy-sensitive.
      *
@@ -597,54 +672,140 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
     private synchronized void enqueueEvent(final String[] keys, final Object[] values) {
         assert values.length + 1 == keys.length;
         if (isAllowedToLog()) {
-            mCurrentLogUnit.addLogStatement(keys, values, false /* isPotentiallyPrivate */);
+            mCurrentLogUnit.addLogAtom(keys, values, false);
         }
     }
 
-    /* package for test */ void commitCurrentLogUnit() {
+    // Used to track how often words are logged.  Too-frequent logging can leak
-        if (!mCurrentLogUnit.isEmpty()) {
+    // semantics, disclosing private data.
-            if (mMainLogBuffer != null) {
+    /* package for test */ static class LoggingFrequencyState {
-                mMainLogBuffer.shiftIn(mCurrentLogUnit);
+        private static final int DEFAULT_WORD_LOG_FREQUENCY = 10;
-                if (mMainLogBuffer.isSafeToLog() && mMainResearchLog != null) {
+        private int mWordsRemainingToSkip;
-                    publishLogBuffer(mMainLogBuffer, mMainResearchLog,
+        private final int mFrequency;
-                            true /* isIncludingPrivateData */);
+
-                    mMainLogBuffer.resetWordCounter();
+        /**
+         * Tracks how often words may be uploaded.
+         *
+         * @param frequency 1=Every word, 2=Every other word, etc.
+         */
+        public LoggingFrequencyState(int frequency) {
+            mFrequency = frequency;
+            mWordsRemainingToSkip = mFrequency;
+        }
+
+        public void onWordLogged() {
+            mWordsRemainingToSkip = mFrequency;
+        }
+
+        public void onWordNotLogged() {
+            if (mWordsRemainingToSkip > 1) {
+                mWordsRemainingToSkip--;
+            }
+        }
+
+        public boolean isSafeToLog() {
+            return mWordsRemainingToSkip <= 1;
+        }
+    }
+
+    /* package for test */ LoggingFrequencyState mLoggingFrequencyState =
+            new LoggingFrequencyState(LoggingFrequencyState.DEFAULT_WORD_LOG_FREQUENCY);
+
+    /* package for test */ boolean isPrivacyThreat(String word) {
+        // Current checks:
+        // - Word not in dictionary
+        // - Word contains numbers
+        // - Privacy-safe word not logged recently
+        if (TextUtils.isEmpty(word)) {
+            return false;
+        }
+        if (!mLoggingFrequencyState.isSafeToLog()) {
+            return true;
+        }
+        final int length = word.length();
+        boolean hasLetter = false;
+        for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) {
+            final int codePoint = Character.codePointAt(word, i);
+            if (Character.isDigit(codePoint)) {
+                return true;
+            }
+            if (Character.isLetter(codePoint)) {
+                hasLetter = true;
+                break; // Word may contain digits, but will only be allowed if in the dictionary.
+            }
+        }
+        if (hasLetter) {
+            if (mDictionary == null && mSuggest != null && mSuggest.hasMainDictionary()) {
+                mDictionary = mSuggest.getMainDictionary();
+            }
+            if (mDictionary == null) {
+                // Can't access dictionary.  Assume privacy threat.
+                return true;
+            }
+            return !(mDictionary.isValidWord(word));
+        }
+        // No letters, no numbers.  Punctuation, space, or something else.
+        return false;
+    }
+
+    private void onWordComplete(String word) {
+        if (isPrivacyThreat(word)) {
+            publishLogUnit(mCurrentLogUnit, true);
+            mLoggingFrequencyState.onWordNotLogged();
+        } else {
+            publishLogUnit(mCurrentLogUnit, false);
+            mLoggingFrequencyState.onWordLogged();
+        }
+        mCurrentLogUnit = new LogUnit();
+        mStatistics.recordWordEntered();
+    }
+
+    private void publishLogUnit(LogUnit logUnit, boolean isPrivacySensitive) {
+        if (!isAllowedToLog()) {
+            return;
+        }
+        if (mMainResearchLog == null) {
+            return;
+        }
+        if (isPrivacySensitive) {
+            mMainResearchLog.publishPublicEvents(logUnit);
+        } else {
+            mMainResearchLog.publishAllEvents(logUnit);
+        }
+        mIntentionalResearchLogQueue.add(logUnit);
+    }
+
+    /* package */ void publishCurrentLogUnit(ResearchLog researchLog, boolean isPrivacySensitive) {
+        publishLogUnit(mCurrentLogUnit, isPrivacySensitive);
+    }
+
+    static class LogUnit {
+        private final List<String[]> mKeysList = new ArrayList<String[]>();
+        private final List<Object[]> mValuesList = new ArrayList<Object[]>();
+        private final List<Boolean> mIsPotentiallyPrivate = new ArrayList<Boolean>();
+
+        private void addLogAtom(final String[] keys, final Object[] values,
+                final Boolean isPotentiallyPrivate) {
+            mKeysList.add(keys);
+            mValuesList.add(values);
+            mIsPotentiallyPrivate.add(isPotentiallyPrivate);
+        }
+
+        public void publishPublicEventsTo(ResearchLog researchLog) {
+            final int size = mKeysList.size();
+            for (int i = 0; i < size; i++) {
+                if (!mIsPotentiallyPrivate.get(i)) {
+                    researchLog.outputEvent(mKeysList.get(i), mValuesList.get(i));
                 }
             }
-            if (mFeedbackLogBuffer != null) {
-                mFeedbackLogBuffer.shiftIn(mCurrentLogUnit);
-            }
-            mCurrentLogUnit = new LogUnit();
-            Log.d(TAG, "commitCurrentLogUnit");
         }
-    }
 
-    /* package for test */ void publishLogBuffer(final LogBuffer logBuffer,
+        public void publishAllEventsTo(ResearchLog researchLog) {
-            final ResearchLog researchLog, final boolean isIncludingPrivateData) {
+            final int size = mKeysList.size();
-        LogUnit logUnit;
+            for (int i = 0; i < size; i++) {
-        while ((logUnit = logBuffer.shiftOut()) != null) {
+                researchLog.outputEvent(mKeysList.get(i), mValuesList.get(i));
-            researchLog.publish(logUnit, isIncludingPrivateData);
-        }
-    }
-
-    private boolean hasOnlyLetters(final String word) {
-        final int length = word.length();
-        for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) {
-            final int codePoint = word.codePointAt(i);
-            if (!Character.isLetter(codePoint)) {
-                return false;
             }
         }
-        return true;
-    }
-
-    private void onWordComplete(final String word) {
-        Log.d(TAG, "onWordComplete: " + word);
-        if (word != null && word.length() > 0 && hasOnlyLetters(word)) {
-            mCurrentLogUnit.setWord(word);
-            mStatistics.recordWordEntered();
-        }
-        commitCurrentLogUnit();
     }
 
     private static int scrubDigitFromCodePoint(int codePoint) {
@@ -697,6 +858,12 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
         return WORD_REPLACEMENT_STRING;
     }
 
+    // Special methods related to startup, shutdown, logging itself
+
+    private static final String[] EVENTKEYS_INTENTIONAL_LOG = {
+        "IntentionalLog"
+    };
+
     private static final String[] EVENTKEYS_LATINIME_ONSTARTINPUTVIEWINTERNAL = {
         "LatinIMEOnStartInputViewInternal", "uuid", "packageName", "inputType", "imeOptions",
         "fieldId", "display", "model", "prefs", "versionCode", "versionName", "outputFormatVersion"
@@ -704,6 +871,9 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
     public static void latinIME_onStartInputViewInternal(final EditorInfo editorInfo,
             final SharedPreferences prefs) {
         final ResearchLogger researchLogger = getInstance();
+        if (researchLogger.mInFeedbackDialog) {
+            researchLogger.saveLogsForFeedback();
+        }
         researchLogger.start();
         if (editorInfo != null) {
             final Context context = researchLogger.mInputMethodService;
@@ -735,6 +905,16 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
         "UserFeedback", "FeedbackContents"
     };
 
+    private void userFeedback(ResearchLog researchLog, String feedbackContents) {
+        // this method is special; it directs the feedbackContents to a particular researchLog
+        final LogUnit logUnit = new LogUnit();
+        final Object[] values = {
+            feedbackContents
+        };
+        logUnit.addLogAtom(EVENTKEYS_USER_FEEDBACK, values, false);
+        researchLog.publishAllEvents(logUnit);
+    }
+
     // Regular logging methods
 
     private static final String[] EVENTKEYS_MAINKEYBOARDVIEW_PROCESSMOTIONEVENT = {
@@ -769,16 +949,12 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
         "LatinIMEOnCodeInput", "code", "x", "y"
     };
     public static void latinIME_onCodeInput(final int code, final int x, final int y) {
-        final long time = SystemClock.uptimeMillis();
-        final ResearchLogger researchLogger = getInstance();
         final Object[] values = {
             Keyboard.printableCode(scrubDigitFromCodePoint(code)), x, y
         };
+        final ResearchLogger researchLogger = getInstance();
         researchLogger.enqueuePotentiallyPrivateEvent(EVENTKEYS_LATINIME_ONCODEINPUT, values);
-        if (Character.isDigit(code)) {
+        researchLogger.mStatistics.recordChar(code, SystemClock.uptimeMillis());
-            researchLogger.setCurrentLogUnitContainsDigitFlag();
-        }
-        researchLogger.mStatistics.recordChar(code, time);
     }
 
     private static final String[] EVENTKEYS_LATINIME_ONDISPLAYCOMPLETIONS = {
@@ -843,7 +1019,9 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
             }
             final ResearchLogger researchLogger = getInstance();
             researchLogger.enqueueEvent(EVENTKEYS_LATINIME_ONWINDOWHIDDEN, values);
-            researchLogger.commitCurrentLogUnit();
+            // Play it safe.  Remove privacy-sensitive events.
+            researchLogger.publishLogUnit(researchLogger.mCurrentLogUnit, true);
+            researchLogger.mCurrentLogUnit = new LogUnit();
             getInstance().stop();
         }
     }
@@ -908,11 +1086,7 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
         final Object[] values = {
             Keyboard.printableCode(scrubDigitFromCodePoint(code))
         };
-        final ResearchLogger researchLogger = getInstance();
+        getInstance().enqueuePotentiallyPrivateEvent(EVENTKEYS_LATINIME_SENDKEYCODEPOINT, values);
-        researchLogger.enqueuePotentiallyPrivateEvent(EVENTKEYS_LATINIME_SENDKEYCODEPOINT, values);
-        if (Character.isDigit(code)) {
-            researchLogger.setCurrentLogUnitContainsDigitFlag();
-        }
     }
 
     private static final String[] EVENTKEYS_LATINIME_SWAPSWAPPERANDSPACE = {
@@ -1050,21 +1224,10 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
                 EVENTKEYS_RICHINPUTCONNECTION_COMMITCOMPLETION, values);
     }
 
-    // Disabled for privacy-protection reasons.  Because this event comes after
-    // richInputConnection_commitText, which is the event used to separate LogUnits, the
-    // data in this event can be associated with the next LogUnit, revealing information
-    // about the current word even if it was supposed to be suppressed.  The occurrance of
-    // autocorrection can be determined by examining the difference between the text strings in
-    // the last call to richInputConnection_setComposingText before
-    // richInputConnection_commitText, so it's not a data loss.
-    // TODO: Figure out how to log this event without loss of privacy.
-    /*
     private static final String[] EVENTKEYS_RICHINPUTCONNECTION_COMMITCORRECTION = {
-        "RichInputConnectionCommitCorrection", "typedWord", "autoCorrection"
+        "RichInputConnectionCommitCorrection", "CorrectionInfo"
     };
-    */
     public static void richInputConnection_commitCorrection(CorrectionInfo correctionInfo) {
-        /*
         final String typedWord = correctionInfo.getOldText().toString();
         final String autoCorrection = correctionInfo.getNewText().toString();
         final Object[] values = {
@@ -1073,7 +1236,6 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
         final ResearchLogger researchLogger = getInstance();
         researchLogger.enqueuePotentiallyPrivateEvent(
                 EVENTKEYS_RICHINPUTCONNECTION_COMMITCORRECTION, values);
-        */
     }
 
     private static final String[] EVENTKEYS_RICHINPUTCONNECTION_COMMITTEXT = {
@@ -1099,8 +1261,7 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
         final Object[] values = {
             beforeLength, afterLength
         };
-        getInstance().enqueuePotentiallyPrivateEvent(
+        getInstance().enqueueEvent(EVENTKEYS_RICHINPUTCONNECTION_DELETESURROUNDINGTEXT, values);
-                EVENTKEYS_RICHINPUTCONNECTION_DELETESURROUNDINGTEXT, values);
     }
 
     private static final String[] EVENTKEYS_RICHINPUTCONNECTION_FINISHCOMPOSINGTEXT = {
@@ -1130,8 +1291,7 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
             keyEvent.getAction(),
             keyEvent.getKeyCode()
         };
-        getInstance().enqueuePotentiallyPrivateEvent(EVENTKEYS_RICHINPUTCONNECTION_SENDKEYEVENT,
+        getInstance().enqueueEvent(EVENTKEYS_RICHINPUTCONNECTION_SENDKEYEVENT, values);
-                values);
     }
 
     private static final String[] EVENTKEYS_RICHINPUTCONNECTION_SETCOMPOSINGTEXT = {
@@ -1139,14 +1299,10 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
     };
     public static void richInputConnection_setComposingText(final CharSequence text,
             final int newCursorPosition) {
-        if (text == null) {
-            throw new RuntimeException("setComposingText is null");
-        }
         final Object[] values = {
             text, newCursorPosition
         };
-        getInstance().enqueuePotentiallyPrivateEvent(EVENTKEYS_RICHINPUTCONNECTION_SETCOMPOSINGTEXT,
+        getInstance().enqueueEvent(EVENTKEYS_RICHINPUTCONNECTION_SETCOMPOSINGTEXT, values);
-                values);
     }
 
     private static final String[] EVENTKEYS_RICHINPUTCONNECTION_SETSELECTION = {
@@ -1156,8 +1312,7 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
         final Object[] values = {
             from, to
         };
-        getInstance().enqueuePotentiallyPrivateEvent(EVENTKEYS_RICHINPUTCONNECTION_SETSELECTION,
+        getInstance().enqueueEvent(EVENTKEYS_RICHINPUTCONNECTION_SETSELECTION, values);
-                values);
     }
 
     private static final String[] EVENTKEYS_SUDDENJUMPINGTOUCHEVENTHANDLER_ONTOUCHEVENT = {
@@ -1192,24 +1347,4 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
     public void userTimestamp() {
         getInstance().enqueueEvent(EVENTKEYS_USER_TIMESTAMP, EVENTKEYS_NULLVALUES);
     }
-
-    private static final String[] EVENTKEYS_STATISTICS = {
-        "Statistics", "charCount", "letterCount", "numberCount", "spaceCount", "deleteOpsCount",
-        "wordCount", "isEmptyUponStarting", "isEmptinessStateKnown", "averageTimeBetweenKeys",
-        "averageTimeBeforeDelete", "averageTimeDuringRepeatedDelete", "averageTimeAfterDelete"
-    };
-    private static void logStatistics() {
-        final ResearchLogger researchLogger = getInstance();
-        final Statistics statistics = researchLogger.mStatistics;
-        final Object[] values = {
-                statistics.mCharCount, statistics.mLetterCount, statistics.mNumberCount,
-                statistics.mSpaceCount, statistics.mDeleteKeyCount,
-                statistics.mWordCount, statistics.mIsEmptyUponStarting,
-                statistics.mIsEmptinessStateKnown, statistics.mKeyCounter.getAverageTime(),
-                statistics.mBeforeDeleteKeyCounter.getAverageTime(),
-                statistics.mDuringRepeatedDeleteKeysCounter.getAverageTime(),
-                statistics.mAfterDeleteKeyCounter.getAverageTime()
-        };
-        researchLogger.enqueueEvent(EVENTKEYS_STATISTICS, values);
-    }
 }

java/src/com/android/inputmethod/research/Statistics.java

@@ -66,8 +66,8 @@ public class Statistics {
 
     // To account for the interruptions when the user's attention is directed elsewhere, times
     // longer than MIN_TYPING_INTERMISSION are not counted when estimating this statistic.
-    public static final int MIN_TYPING_INTERMISSION = 2 * 1000;  // in milliseconds
+    public static final int MIN_TYPING_INTERMISSION = 5 * 1000;  // in milliseconds
-    public static final int MIN_DELETION_INTERMISSION = 10 * 1000;  // in milliseconds
+    public static final int MIN_DELETION_INTERMISSION = 15 * 1000;  // in milliseconds
 
     // The last time that a tap was performed
     private long mLastTapTime;