From 7708bcf6fb80f42f62f34b57aece4a2baa5b3320 Mon Sep 17 00:00:00 2001 From: Kurt Partridge Date: Thu, 31 Jan 2013 09:59:16 -0800 Subject: [PATCH] [Rlog48c] Replay historical motion data Change-Id: Ib398ea61ff048b1a4ac3b7f7b4a772e173a7b294 --- .../research/MotionEventReader.java | 253 ++++++++++++++++-- .../inputmethod/research/Replayer.java | 20 +- .../research/MotionEventReaderTests.java | 169 ++++++++++++ 3 files changed, 421 insertions(+), 21 deletions(-) create mode 100644 tests/src/com/android/inputmethod/research/MotionEventReaderTests.java diff --git a/java/src/com/android/inputmethod/research/MotionEventReader.java b/java/src/com/android/inputmethod/research/MotionEventReader.java index 26a1d7f55..e59adfa19 100644 --- a/java/src/com/android/inputmethod/research/MotionEventReader.java +++ b/java/src/com/android/inputmethod/research/MotionEventReader.java @@ -19,6 +19,8 @@ package com.android.inputmethod.research; import android.util.JsonReader; import android.util.Log; import android.view.MotionEvent; +import android.view.MotionEvent.PointerCoords; +import android.view.MotionEvent.PointerProperties; import com.android.inputmethod.latin.define.ProductionFlag; @@ -33,6 +35,14 @@ import java.util.ArrayList; public class MotionEventReader { private static final String TAG = MotionEventReader.class.getSimpleName(); private static final boolean DEBUG = false && ProductionFlag.IS_EXPERIMENTAL_DEBUG; + // Assumes that MotionEvent.ACTION_MASK does not have all bits set.` + private static final int UNINITIALIZED_ACTION = ~MotionEvent.ACTION_MASK; + // No legitimate int is negative + private static final int UNINITIALIZED_INT = -1; + // No legitimate long is negative + private static final long UNINITIALIZED_LONG = -1L; + // No legitimate float is negative + private static final float UNINITIALIZED_FLOAT = -1.0f; public ReplayData readMotionEventData(final File file) { final ReplayData replayData = new ReplayData(); @@ -55,19 +65,82 @@ public class MotionEventReader { static class ReplayData { final ArrayList mActions = new ArrayList(); - final ArrayList mXCoords = new ArrayList(); - final ArrayList mYCoords = new ArrayList(); + final ArrayList mPointerPropertiesArrays + = new ArrayList(); + final ArrayList mPointerCoordsArrays = new ArrayList(); final ArrayList mTimes = new ArrayList(); } - private void readLogStatement(final JsonReader jsonReader, final ReplayData replayData) - throws IOException { + /** + * Read motion data from a logStatement and store it in {@code replayData}. + * + * Two kinds of logStatements can be read. In the first variant, the MotionEvent data is + * represented as attributes at the top level like so: + * + * 
+     * {
+     *   "_ct": 1359590400000,
+     *   "_ut": 4381933,
+     *   "_ty": "MotionEvent",
+     *   "action": "UP",
+     *   "isLoggingRelated": false,
+     *   "x": 100,
+     *   "y": 200
+     * }
+     *
+ * + * In the second variant, there is a separate attribute for the MotionEvent that includes + * historical data if present: + * + * 
+     * {
+     *   "_ct": 135959040000,
+     *   "_ut": 4382702,
+     *   "_ty": "MotionEvent",
+     *   "action": "MOVE",
+     *   "isLoggingRelated": false,
+     *   "motionEvent": {
+     *     "pointerIds": [
+     *       0
+     *     ],
+     *     "xyt": [
+     *       {
+     *         "t": 4382551,
+     *         "d": [
+     *           {
+     *             "x": 141.25,
+     *             "y": 151.8485107421875,
+     *             "toma": 101.82337188720703,
+     *             "tomi": 101.82337188720703,
+     *             "o": 0.0
+     *           }
+     *         ]
+     *       },
+     *       {
+     *         "t": 4382559,
+     *         "d": [
+     *           {
+     *             "x": 140.7266082763672,
+     *             "y": 151.8485107421875,
+     *             "toma": 101.82337188720703,
+     *             "tomi": 101.82337188720703,
+     *             "o": 0.0
+     *           }
+     *         ]
+     *       }
+     *     ]
+     *   }
+     * },
+     *
+ */ + /* package for test */ void readLogStatement(final JsonReader jsonReader, + final ReplayData replayData) throws IOException { String logStatementType = null; - Integer actionType = null; - Integer x = null; - Integer y = null; - Long time = null; - boolean loggingRelated = false; + int actionType = UNINITIALIZED_ACTION; + int x = UNINITIALIZED_INT; + int y = UNINITIALIZED_INT; + long time = UNINITIALIZED_LONG; + boolean isLoggingRelated = false; jsonReader.beginObject(); while (jsonReader.hasNext()) { @@ -90,7 +163,18 @@ public class MotionEventReader { actionType = MotionEvent.ACTION_MOVE; } } else if (key.equals("loggingRelated")) { - loggingRelated = jsonReader.nextBoolean(); + isLoggingRelated = jsonReader.nextBoolean(); + } else if (logStatementType != null && logStatementType.equals("MotionEvent") + && key.equals("motionEvent")) { + if (actionType == UNINITIALIZED_ACTION) { + Log.e(TAG, "no actionType assigned in MotionEvent json"); + } + // Second variant of LogStatement. + if (isLoggingRelated) { + jsonReader.skipValue(); + } else { + readEmbeddedMotionEvent(jsonReader, replayData, actionType); + } } else { if (DEBUG) { Log.w(TAG, "Unknown JSON key in LogStatement: " + key); @@ -100,14 +184,149 @@ public class MotionEventReader { } jsonReader.endObject(); - if (logStatementType != null && time != null && x != null && y != null && actionType != null - && logStatementType.equals("MotionEvent") - && !loggingRelated) { - replayData.mActions.add(actionType); - replayData.mXCoords.add(x); - replayData.mYCoords.add(y); - replayData.mTimes.add(time); + if (logStatementType != null && time != UNINITIALIZED_LONG && x != UNINITIALIZED_INT + && y != UNINITIALIZED_INT && actionType != UNINITIALIZED_ACTION + && logStatementType.equals("MotionEvent") && !isLoggingRelated) { + // First variant of LogStatement. + final PointerProperties pointerProperties = new PointerProperties(); + pointerProperties.id = 0; + pointerProperties.toolType = MotionEvent.TOOL_TYPE_UNKNOWN; + final PointerProperties[] pointerPropertiesArray = { + pointerProperties + }; + final PointerCoords pointerCoords = new PointerCoords(); + pointerCoords.x = x; + pointerCoords.y = y; + pointerCoords.pressure = 1.0f; + pointerCoords.size = 1.0f; + final PointerCoords[] pointerCoordsArray = { + pointerCoords + }; + addMotionEventData(replayData, actionType, time, pointerPropertiesArray, + pointerCoordsArray); } } + private void readEmbeddedMotionEvent(final JsonReader jsonReader, final ReplayData replayData, + final int actionType) throws IOException { + jsonReader.beginObject(); + PointerProperties[] pointerPropertiesArray = null; + while (jsonReader.hasNext()) { // pointerIds/xyt + final String name = jsonReader.nextName(); + if (name.equals("pointerIds")) { + pointerPropertiesArray = readPointerProperties(jsonReader); + } else if (name.equals("xyt")) { + readPointerData(jsonReader, replayData, actionType, pointerPropertiesArray); + } + } + jsonReader.endObject(); + } + + private PointerProperties[] readPointerProperties(final JsonReader jsonReader) + throws IOException { + final ArrayList pointerPropertiesArrayList = + new ArrayList(); + jsonReader.beginArray(); + while (jsonReader.hasNext()) { + final PointerProperties pointerProperties = new PointerProperties(); + pointerProperties.id = jsonReader.nextInt(); + pointerProperties.toolType = MotionEvent.TOOL_TYPE_UNKNOWN; + pointerPropertiesArrayList.add(pointerProperties); + } + jsonReader.endArray(); + return pointerPropertiesArrayList.toArray( + new PointerProperties[pointerPropertiesArrayList.size()]); + } + + private void readPointerData(final JsonReader jsonReader, final ReplayData replayData, + final int actionType, final PointerProperties[] pointerPropertiesArray) + throws IOException { + if (pointerPropertiesArray == null) { + Log.e(TAG, "PointerIDs must be given before xyt data in json for MotionEvent"); + jsonReader.skipValue(); + return; + } + long time = UNINITIALIZED_LONG; + jsonReader.beginArray(); + while (jsonReader.hasNext()) { // Array of historical data + jsonReader.beginObject(); + final ArrayList pointerCoordsArrayList = new ArrayList(); + while (jsonReader.hasNext()) { // Time/data object + final String name = jsonReader.nextName(); + if (name.equals("t")) { + time = jsonReader.nextLong(); + } else if (name.equals("d")) { + jsonReader.beginArray(); + while (jsonReader.hasNext()) { // array of data per pointer + final PointerCoords pointerCoords = readPointerCoords(jsonReader); + if (pointerCoords != null) { + pointerCoordsArrayList.add(pointerCoords); + } + } + jsonReader.endArray(); + } else { + jsonReader.skipValue(); + } + } + jsonReader.endObject(); + // Data was recorded as historical events, but must be split apart into + // separate MotionEvents for replaying + if (time != UNINITIALIZED_LONG) { + addMotionEventData(replayData, actionType, time, pointerPropertiesArray, + pointerCoordsArrayList.toArray( + new PointerCoords[pointerCoordsArrayList.size()])); + } else { + Log.e(TAG, "Time not assigned in json for MotionEvent"); + } + } + jsonReader.endArray(); + } + + private PointerCoords readPointerCoords(final JsonReader jsonReader) throws IOException { + jsonReader.beginObject(); + float x = UNINITIALIZED_FLOAT; + float y = UNINITIALIZED_FLOAT; + while (jsonReader.hasNext()) { // x,y + final String name = jsonReader.nextName(); + if (name.equals("x")) { + x = (float) jsonReader.nextDouble(); + } else if (name.equals("y")) { + y = (float) jsonReader.nextDouble(); + } else { + jsonReader.skipValue(); + } + } + jsonReader.endObject(); + + if (Float.compare(x, UNINITIALIZED_FLOAT) == 0 + || Float.compare(y, UNINITIALIZED_FLOAT) == 0) { + Log.w(TAG, "missing x or y value in MotionEvent json"); + return null; + } + final PointerCoords pointerCoords = new PointerCoords(); + pointerCoords.x = x; + pointerCoords.y = y; + pointerCoords.pressure = 1.0f; + pointerCoords.size = 1.0f; + return pointerCoords; + } + + /** + * Tests that {@code x} is uninitialized. + * + * Assumes that {@code x} will never be given a valid value less than 0, and that + * UNINITIALIZED_FLOAT is less than 0.0f. + */ + private boolean isUninitializedFloat(final float x) { + return x < 0.0f; + } + + private void addMotionEventData(final ReplayData replayData, final int actionType, + final long time, final PointerProperties[] pointerProperties, + final PointerCoords[] pointerCoords) { + replayData.mActions.add(actionType); + replayData.mTimes.add(time); + replayData.mPointerPropertiesArrays.add(pointerProperties); + replayData.mPointerCoordsArrays.add(pointerCoords); + } } diff --git a/java/src/com/android/inputmethod/research/Replayer.java b/java/src/com/android/inputmethod/research/Replayer.java index 9bf5fee6c..a9b7a9d0c 100644 --- a/java/src/com/android/inputmethod/research/Replayer.java +++ b/java/src/com/android/inputmethod/research/Replayer.java @@ -22,6 +22,8 @@ import android.os.Message; import android.os.SystemClock; import android.util.Log; import android.view.MotionEvent; +import android.view.MotionEvent.PointerCoords; +import android.view.MotionEvent.PointerProperties; import com.android.inputmethod.keyboard.KeyboardSwitcher; import com.android.inputmethod.keyboard.MainKeyboardView; @@ -62,7 +64,6 @@ public class Replayer { if (mIsReplaying) { return; } - mIsReplaying = true; final int numActions = replayData.mActions.size(); if (DEBUG) { @@ -95,25 +96,36 @@ public class Replayer { case MSG_MOTION_EVENT: final int index = msg.arg1; final int action = replayData.mActions.get(index); - final int x = replayData.mXCoords.get(index); - final int y = replayData.mYCoords.get(index); + final PointerProperties[] pointerPropertiesArray = + replayData.mPointerPropertiesArrays.get(index); + final PointerCoords[] pointerCoordsArray = + replayData.mPointerCoordsArrays.get(index); final long origTime = replayData.mTimes.get(index); if (action == MotionEvent.ACTION_DOWN) { mOrigDownTime = origTime; } final MotionEvent me = MotionEvent.obtain(mOrigDownTime + timeAdjustment, - origTime + timeAdjustment, action, x, y, 0); + origTime + timeAdjustment, action, + pointerPropertiesArray.length, pointerPropertiesArray, + pointerCoordsArray, 0, 0, 1.0f, 1.0f, 0, 0, 0, 0); mainKeyboardView.processMotionEvent(me); me.recycle(); break; case MSG_DONE: mIsReplaying = false; + ResearchLogger.getInstance().requestIndicatorRedraw(); break; } } }; + handler.post(new Runnable() { + @Override + public void run() { + ResearchLogger.getInstance().requestIndicatorRedraw(); + } + }); for (int i = 0; i < numActions; i++) { final Message msg = Message.obtain(handler, MSG_MOTION_EVENT, i, 0); final long msgTime = replayData.mTimes.get(i) + timeAdjustment; diff --git a/tests/src/com/android/inputmethod/research/MotionEventReaderTests.java b/tests/src/com/android/inputmethod/research/MotionEventReaderTests.java new file mode 100644 index 000000000..c0eaaead4 --- /dev/null +++ b/tests/src/com/android/inputmethod/research/MotionEventReaderTests.java @@ -0,0 +1,169 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.android.inputmethod.research; + +import android.test.AndroidTestCase; +import android.util.JsonReader; + +import com.android.inputmethod.research.MotionEventReader.ReplayData; + +import java.io.IOException; +import java.io.StringReader; + +public class MotionEventReaderTests extends AndroidTestCase { + private MotionEventReader mMotionEventReader = new MotionEventReader(); + private ReplayData mReplayData; + + @Override + protected void setUp() throws Exception { + super.setUp(); + mReplayData = new ReplayData(); + } + + private JsonReader jsonReaderForString(final String s) { + return new JsonReader(new StringReader(s)); + } + + public void testTopLevelDataVariant() { + final JsonReader jsonReader = jsonReaderForString( + "{" + + "\"_ct\": 1359590400000," + + "\"_ut\": 4381933," + + "\"_ty\": \"MotionEvent\"," + + "\"action\": \"UP\"," + + "\"isLoggingRelated\": false," + + "\"x\": 100.0," + + "\"y\": 200.0" + + "}" + ); + try { + mMotionEventReader.readLogStatement(jsonReader, mReplayData); + } catch (IOException e) { + e.printStackTrace(); + fail("IOException thrown"); + } + assertEquals("x set correctly", (int) mReplayData.mPointerCoordsArrays.get(0)[0].x, 100); + assertEquals("y set correctly", (int) mReplayData.mPointerCoordsArrays.get(0)[0].y, 200); + assertEquals("only one pointer", mReplayData.mPointerCoordsArrays.get(0).length, 1); + assertEquals("only one MotionEvent", mReplayData.mPointerCoordsArrays.size(), 1); + } + + public void testNestedDataVariant() { + final JsonReader jsonReader = jsonReaderForString( + "{" + + " \"_ct\": 135959040000," + + " \"_ut\": 4382702," + + " \"_ty\": \"MotionEvent\"," + + " \"action\": \"MOVE\"," + + " \"isLoggingRelated\": false," + + " \"motionEvent\": {" + + " \"pointerIds\": [" + + " 0" + + " ]," + + " \"xyt\": [" + + " {" + + " \"t\": 4382551," + + " \"d\": [" + + " {" + + " \"x\": 100.0," + + " \"y\": 200.0," + + " \"toma\": 999.0," + + " \"tomi\": 999.0," + + " \"o\": 0.0" + + " }" + + " ]" + + " }," + + " {" + + " \"t\": 4382559," + + " \"d\": [" + + " {" + + " \"x\": 300.0," + + " \"y\": 400.0," + + " \"toma\": 999.0," + + " \"tomi\": 999.0," + + " \"o\": 0.0" + + " }" + + " ]" + + " }" + + " ]" + + " }" + + "}" + ); + try { + mMotionEventReader.readLogStatement(jsonReader, mReplayData); + } catch (IOException e) { + e.printStackTrace(); + fail("IOException thrown"); + } + assertEquals("x1 set correctly", (int) mReplayData.mPointerCoordsArrays.get(0)[0].x, 100); + assertEquals("y1 set correctly", (int) mReplayData.mPointerCoordsArrays.get(0)[0].y, 200); + assertEquals("x2 set correctly", (int) mReplayData.mPointerCoordsArrays.get(1)[0].x, 300); + assertEquals("y2 set correctly", (int) mReplayData.mPointerCoordsArrays.get(1)[0].y, 400); + assertEquals("only one pointer", mReplayData.mPointerCoordsArrays.get(0).length, 1); + assertEquals("two MotionEvents", mReplayData.mPointerCoordsArrays.size(), 2); + } + + public void testNestedDataVariantMultiPointer() { + final JsonReader jsonReader = jsonReaderForString( + "{" + + " \"_ct\": 135959040000," + + " \"_ut\": 4382702," + + " \"_ty\": \"MotionEvent\"," + + " \"action\": \"MOVE\"," + + " \"isLoggingRelated\": false," + + " \"motionEvent\": {" + + " \"pointerIds\": [" + + " 1" + + " ]," + + " \"xyt\": [" + + " {" + + " \"t\": 4382551," + + " \"d\": [" + + " {" + + " \"x\": 100.0," + + " \"y\": 200.0," + + " \"toma\": 999.0," + + " \"tomi\": 999.0," + + " \"o\": 0.0" + + " }," + + " {" + + " \"x\": 300.0," + + " \"y\": 400.0," + + " \"toma\": 999.0," + + " \"tomi\": 999.0," + + " \"o\": 0.0" + + " }" + + " ]" + + " }" + + " ]" + + " }" + + "}" + ); + try { + mMotionEventReader.readLogStatement(jsonReader, mReplayData); + } catch (IOException e) { + e.printStackTrace(); + fail("IOException thrown"); + } + assertEquals("x1 set correctly", (int) mReplayData.mPointerCoordsArrays.get(0)[0].x, 100); + assertEquals("y1 set correctly", (int) mReplayData.mPointerCoordsArrays.get(0)[0].y, 200); + assertEquals("x2 set correctly", (int) mReplayData.mPointerCoordsArrays.get(0)[1].x, 300); + assertEquals("y2 set correctly", (int) mReplayData.mPointerCoordsArrays.get(0)[1].y, 400); + assertEquals("two pointers", mReplayData.mPointerCoordsArrays.get(0).length, 2); + assertEquals("one MotionEvent", mReplayData.mPointerCoordsArrays.size(), 1); + } +}