vllm-omni/benchmarks/edge_harness/android_agent/action_space.py

"""Android control action space for a text-only LLM agent.

The tool surface mirrors what the mainstream ADB-based harnesses expose
(droidrun/mobilerun's Portal tools, ghost-in-the-droid's touch/see tools,
AndroidWorld's `JSONAction`), so an agent written against these schemas can be
pointed at any of them. Every action is expressed over the accessibility tree
rather than pixel coordinates, because Spark-X2.5 is a text-only model: it
never sees a screenshot, it sees the uiautomator dump.

`ADB_TEMPLATES` gives the literal `adb` invocation each tool maps to, which is
what a driver executes against a real device.
"""

TOOLS = [
    {
        "type": "function",
        "function": {
            "name": "tap",
            "description": (
                "Tap a UI element. Use the element's index exactly as shown in "
                "the accessibility tree."
            ),
            "parameters": {
                "type": "object",
                "properties": {
                    "index": {
                        "type": "integer",
                        "description": "Index of the element in the tree",
                    }
                },
                "required": ["index"],
            },
        },
    },
    {
        "type": "function",
        "function": {
            "name": "type_text",
            "description": (
                "Type text into a focused input field. Tap the field first."
            ),
            "parameters": {
                "type": "object",
                "properties": {
                    "index": {"type": "integer", "description": "Index of the input field"},
                    "text": {"type": "string", "description": "Text to enter"},
                },
                "required": ["index", "text"],
            },
        },
    },
    {
        "type": "function",
        "function": {
            "name": "swipe",
            "description": (
                "Scroll the screen. The direction is where the content should "
                "move: use 'down' to see what is further down the page."
            ),
            "parameters": {
                "type": "object",
                "properties": {
                    "direction": {
                        "type": "string",
                        "enum": ["up", "down", "left", "right"],
                    }
                },
                "required": ["direction"],
            },
        },
    },
    {
        "type": "function",
        "function": {
            "name": "press_key",
            "description": "Press a hardware or system key.",
            "parameters": {
                "type": "object",
                "properties": {
                    "key": {
                        "type": "string",
                        "enum": ["back", "home", "enter", "delete", "recent"],
                    }
                },
                "required": ["key"],
            },
        },
    },
    {
        "type": "function",
        "function": {
            "name": "open_app",
            "description": "Launch an app by its visible name.",
            "parameters": {
                "type": "object",
                "properties": {"name": {"type": "string"}},
                "required": ["name"],
            },
        },
    },
    {
        "type": "function",
        "function": {
            "name": "task_complete",
            "description": (
                "Call when the user's request has been fully carried out."
            ),
            "parameters": {
                "type": "object",
                "properties": {
                    "summary": {"type": "string", "description": "What was done"}
                },
                "required": ["summary"],
            },
        },
    },
]

# How each tool becomes a real device command. `{bounds}` is filled from the
# element's centre point in the accessibility tree.
ADB_TEMPLATES = {
    "tap": "adb shell input tap {x} {y}",
    "type_text": "adb shell input text {text_escaped}",
    "swipe": "adb shell input swipe {x1} {y1} {x2} {y2} 300",
    "press_key": "adb shell input keyevent {keycode}",
    "open_app": "adb shell monkey -p {package} -c android.intent.category.LAUNCHER 1",
}

KEYCODES = {
    "back": "KEYCODE_BACK",
    "home": "KEYCODE_HOME",
    "enter": "KEYCODE_ENTER",
    "delete": "KEYCODE_DEL",
    "recent": "KEYCODE_APP_SWITCH",
}

SYSTEM_PROMPT = (
    "You control an Android phone. At each step you are given the current "
    "screen as an accessibility tree: one line per element, with its index, "
    "type, text and whether it is clickable. Call exactly one tool per step to "
    "make progress on the user's task. Refer to elements only by the index "
    "shown.\n"
    # Both rules below exist because the model failed them in measurement: it
    # tapped a Wi-Fi switch that already read checked=true, turning it back
    # off, and never called task_complete. A switch's state is on the screen,
    # so the rule can be stated in terms of what it can see.
    "A switch or checkbox shows its state as checked=true or checked=false. "
    "Tapping it flips that state.\n"
    "Before acting, check whether the screen already shows what the task "
    "asked for. If it does, call task_complete immediately and do not tap "
    "anything else -- acting again would undo it."
)