AccessibilityService can't receive onKeyEvent or onGesture

Question

I'm a Chinese student who is doing some research in Android security.(There may be have some grammar errors in my sentences) I wanna use the accessibility API to capture the user’s behaviors such as clicking the keyboard of IME and selecting certain characters.. But when I tried this API, I could only obtain the text of the current window and the actions that the user performs instead of any events from the keyboard. Maybe I missed something. (I can't receive any events from onKeyEvent and onGesture. But I can receive events from onAccessibilityEvent.)

I have created the following basic Accessibility Service.

package com.jack.accessibility;

import java.util.List;

import android.accessibilityservice.AccessibilityService;
import android.accessibilityservice.AccessibilityServiceInfo;
import android.annotation.SuppressLint;
import android.util.Log;
import android.view.KeyEvent;
import android.view.accessibility.AccessibilityEvent;
import android.view.accessibility.AccessibilityNodeInfo;

@SuppressLint("NewApi")
public class MyAccessibilityService extends AccessibilityService {
@Override
public void onAccessibilityEvent(AccessibilityEvent event) {
    String eventText = getTypeName(event.getEventType()) + "===="
            + event.getContentDescription();
    Log.e("------------------------", "-------------------------");
    Log.d("PackageName", event.getPackageName().toString());
    Log.d("EventName", eventText);
    traverseNode(getRootInActiveWindow());
    Log.e("------------------------", "-------------------------");
}

@Override
public void onServiceConnected() {
    Log.e("---------ServiceConnected--------------",
            "------------ServiceConnected------------");
    AccessibilityServiceInfo info = new AccessibilityServiceInfo();
    info.packageNames = new String[] { "com.android.mms" };
    info.eventTypes = AccessibilityEvent.TYPES_ALL_MASK;
    info.notificationTimeout = 100;
    info.feedbackType = AccessibilityServiceInfo.FEEDBACK_SPOKEN;
    info.flags = AccessibilityServiceInfo.FLAG_REQUEST_FILTER_KEY_EVENTS;
    setServiceInfo(info);
}

@Override
public boolean onKeyEvent(KeyEvent event) {
    Log.i("Key", ".....");
    return true;
}

@Override
public boolean onGesture(int gestureId) {
    Log.v("THEIA", String.format("onGesture: [type] %s", gIdToString(gestureId)));
    return false;
}

@Override
public void onInterrupt() {
    Log.e("---------Interrupt--------------",
            "------------Interrupt------------");
}

private void traverseNode(AccessibilityNodeInfo node) {
    if (null == node)
        return;

    final int count = node.getChildCount();
    if (count > 0) {
        for (int i = 0; i < count; i++) {
            AccessibilityNodeInfo childNode = node.getChild(i);
            traverseNode(childNode);
        }
    } else {
        CharSequence text = node.getText();
        Log.d("test", "Node text = " + text);
    }

}
private String getTypeName(int type) {
        switch (type) {
        case AccessibilityEvent.TYPE_TOUCH_EXPLORATION_GESTURE_START:
            return "TYPE_TOUCH_EXPLORATION_GESTURE_START";
        case AccessibilityEvent.TYPE_TOUCH_EXPLORATION_GESTURE_END:
            return "TYPE_TOUCH_EXPLORATION_GESTURE_END";
        case AccessibilityEvent.TYPE_TOUCH_INTERACTION_START:
            return "TYPE_TOUCH_INTERACTION_START";
        case AccessibilityEvent.TYPE_TOUCH_INTERACTION_END:
            return "TYPE_TOUCH_INTERACTION_END";
        case AccessibilityEvent.TYPE_GESTURE_DETECTION_START:
            return "TYPE_GESTURE_DETECTION_START";
        case AccessibilityEvent.TYPE_GESTURE_DETECTION_END:
            return "TYPE_GESTURE_DETECTION_END";
        case AccessibilityEvent.TYPE_VIEW_HOVER_ENTER:
            return "TYPE_VIEW_HOVER_ENTER";
        case AccessibilityEvent.TYPE_VIEW_HOVER_EXIT:
            return "TYPE_VIEW_HOVER_EXIT";
        case AccessibilityEvent.TYPE_VIEW_SCROLLED:
            return "TYPE_VIEW_SCROLLED";
        case AccessibilityEvent.TYPE_VIEW_CLICKED:
            return "TYPE_VIEW_CLICKED";
        case AccessibilityEvent.TYPE_VIEW_LONG_CLICKED:
            return "TYPE_VIEW_LONG_CLICKED";
        case AccessibilityEvent.TYPE_VIEW_FOCUSED:
            return "TYPE_VIEW_FOCUSED";
        case AccessibilityEvent.TYPE_VIEW_SELECTED:
            return "TYPE_VIEW_SELECTED";
        case AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED:
            return "TYPE_VIEW_ACCESSIBILITY_FOCUSED";
        case AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED:
            return "TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED";
        case AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED:
            return "TYPE_WINDOW_STATE_CHANGED";
        case AccessibilityEvent.TYPE_NOTIFICATION_STATE_CHANGED:
            return "TYPE_NOTIFICATION_STATE_CHANGED";
        case AccessibilityEvent.TYPE_ANNOUNCEMENT:
            return "TYPE_ANNOUNCEMENT";
        case AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED:
            return "TYPE_WINDOW_CONTENT_CHANGED";
        case AccessibilityEvent.TYPE_VIEW_TEXT_CHANGED:
            return "TYPE_VIEW_TEXT_CHANGED";
        case AccessibilityEvent.TYPE_VIEW_TEXT_SELECTION_CHANGED:
            return "TYPE_VIEW_TEXT_SELECTION_CHANGED";
        case AccessibilityEvent.TYPE_VIEW_TEXT_TRAVERSED_AT_MOVEMENT_GRANULARITY:
            return "TYPE_VIEW_TEXT_TRAVERSED_AT_MOVEMENT_GRANULARITY";
        }
        return "Unknown";
    }

private String gIdToString(int gID) {
    switch (gID) {
    case 1:
        return "GESTURE_SWIPE_UP";
    case 2:
        return "GESTURE_SWIPE_DOWN";
    case 3:
        return "GESTURE_SWIPE_LEFT";
    case 4:
        return "GESTURE_SWIPE_RIGHT";
    case 5:
        return "GESTURE_SWIPE_LEFT_AND_RIGHT";
    case 6:
        return "GESTURE_SWIPE_RIGHT_AND_LEFT";
    case 7:
        return "GESTURE_SWIPE_UP_AND_DOWN";
    case 8:
        return "GESTURE_SWIPE_DOWN_AND_UP";
    case 9:
        return "GESTURE_SWIPE_LEFT_AND_UP";
    case 10:
        return "GESTURE_SWIPE_LEFT_AND_DOWN";
    case 11:
        return "GESTURE_SWIPE_RIGHT_AND_UP";
    case 12:
        return "GESTURE_SWIPE_RIGHT_AND_DOWN";
    case 13:
        return "GESTURE_SWIPE_UP_AND_LEFT";
    case 14:
        return "GESTURE_SWIPE_UP_AND_RIGHT";
    case 15:
        return "GESTURE_SWIPE_DOWN_AND_LEFT";
    case 16:
        return "GESTURE_SWIPE_DOWN_AND_RIGHT";
    }
    return "UNKNOWN";
}
}

AndroidManifest.XML:

<uses-sdk
    android:minSdkVersion="8"
    android:targetSdkVersion="18" />

<uses-permission android:name="android.permission.SYSTEM_ALERT_WINDOW" />
<uses-permission android:name="android.permission.BIND_ACCESSIBILITY_SERVICE" />
<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE"/>
<uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE"/>

<application
    android:allowBackup="true"
    android:icon="@drawable/ic_launcher"
    android:label="@string/app_name"
    android:theme="@style/AppTheme" >
    <activity
        android:name="com.jack.accessibility.MainActivity"
        android:label="@string/app_name" >
        <intent-filter>
            <action android:name="android.intent.action.MAIN" />
            <category android:name="android.intent.category.LAUNCHER" />
        </intent-filter>
    </activity>
    <service android:label="@string/acc_service_name" android:name=".MyAccessibilityService" android:permission="android.permission.BIND_ACCESSIBILITY_SERVICE">
        <intent-filter>
            <action android:name="android.accessibilityservice.AccessibilityService" />
        </intent-filter>
        <meta-data 
            android:name="android.accessibilityservice" 
            android:resource="@xml/phone_accessibility" />
    </service>

</application>

phone_accessibility.xml:

<?xml version="1.0" encoding="utf-8"?>
<accessibility-service android:description="@string/accessibility_service_description"
     android:accessibilityEventTypes="typeAllMask" 
     android:accessibilityFeedbackType="feedbackAllMask" 
     android:notificationTimeout="100" 
     android:canRetrieveWindowContent="true"
     android:canRequestTouchExplorationMode="true"
     android:canRequestEnhancedWebAccessibility="true"
     android:canRequestFilterKeyEvents="true"
     android:accessibilityFlags="flagDefault"
  xmlns:android="http://schemas.android.com/apk/res/android" />

Answer 1

Try to configure the Accessibility Service like this:

<?xml version="1.0" encoding="utf-8"?>
<accessibility-service
  xmlns:android="http://schemas.android.com/apk/res/android"
  android:accessibilityEventTypes="typeContextClicked|typeViewClicked"
  android:packageNames="com.example.andres.eventcapture"
  android:accessibilityFlags="flagRequestFilterKeyEvents"
  android:accessibilityFeedbackType="feedbackAllMask"
  android:notificationTimeout="50"
  android:canRetrieveWindowContent="true"
  android:settingsActivity=""
  android:canRequestFilterKeyEvents="true"
/>

It worked good!

Answer 2

From AccessibilityService, i couldn't get the AccessibilityNodeInfo for the IME or Keyboard.

From what i understood, you are trying to log the keys; but this is not possible. You need to have your own keyboard in order to log the keys pressed on the Keyboard.

But from AccessibilityService, you can get text content from all EditText's except the Password EditText's (who have password as their inputType).

Since you have already figured that out, i won't write the code to get text from a node.

If this was possible, would have been a major security issue wouldn't it?!

Answer 3

Add this:

info.flags = AccessibilityServiceInfo.FLAG_REQUEST_TOUCH_EXPLORATION_MODE;