LeapRobot & Embodied Intelligence

pyproject.toml

@@ -70,6 +70,19 @@ web = ["trafilatura>=2.2"]
 # default `--all-extras` syncs deliberately exclude it (`--no-extra leapspace`,
 # see Makefile/CI/setup.sh); `make space-sync` opts back in.
 leapspace = ["pyqt6>=6.7", "cua-sandbox", "pydantic>=2"]
+robot = [
+    "torch>=2.0",
+    "torchvision>=0.15",
+    "safetensors>=0.4",
+    "huggingface-hub>=0.20",
+    "numpy>=1.24",
+    "opencv-python>=4.8",
+    "pyserial>=3.5",
+    "Pillow>=10.0",
+]
+robot-ros2 = [
+    "rclpy>=3.0; python_version>='3.11'",
+]
 
 [project.scripts]
 leap = "leapflow.__main__:main"

src/leapflow/cli/commands/interactive.py

@@ -1704,11 +1704,33 @@ async def _handle_teach(
         if ctx.session and ctx.session.mode == SessionMode.LEARNING:
             console.warning("Already in teaching mode. Say '/teach stop' to end.")
             return True
-        goal = ""
+
+        # Parse --mode for physical demonstration modes.
+        raw_args = ""
         if line.startswith("teach start "):
-            goal = line[len("teach start "):]
+            raw_args = line[len("teach start "):]
         elif line.startswith("教学开始 "):
-            goal = line[len("教学开始 "):]
+            raw_args = line[len("教学开始 "):]
+
+        from leapflow.cli.commands.slash_handlers import _parse_teach_start_args
+
+        parsed = _parse_teach_start_args(raw_args)
+        mode = parsed.get("mode", "gui")
+
+        if mode in ("teleop", "kinesthetic"):
+            from leapflow.cli.commands.slash_handlers import (
+                _start_physical_teach,
+            )
+
+            result = await _start_physical_teach(ctx, mode, parsed)
+            if result.get("ok"):
+                console.success(result["message"])
+            else:
+                console.error(result.get("message", "Failed to start physical teach."))
+            return True
+
+        # Default GUI path — unchanged.
+        goal = parsed.get("goal", "")
         try:
             session = await ctx.session.enter_learning(goal=goal)
             console.success(f"Teaching started — session {session.session_id}")
@@ -1723,6 +1745,14 @@ async def _handle_teach(
         return True
 
     if line == "teach stop":
+        # Check for an active physical session first.
+        from leapflow.cli.commands.slash_handlers import _stop_physical_teach
+
+        physical_result = await _stop_physical_teach(ctx)
+        if physical_result is not None:
+            console.success(physical_result["message"])
+            return True
+
         if not ctx.session or ctx.session.mode != SessionMode.LEARNING:
             if learning:
                 ctx.imitation.end_control_input()

src/leapflow/cli/commands/registry.py

@@ -96,7 +96,7 @@ def supports_runtime(self, runtime: CommandRuntime) -> bool:
     CommandDef("orient", "Show the agent's unified orientation (layered) and pending re-entries", "Chat", effect=CommandEffect.READ_ONLY),
 
     # Teaching
-    CommandDef("teach start", "Start teaching mode", "Teaching", aliases=("teach",), args_hint="[goal]"),
+    CommandDef("teach start", "Start teaching mode", "Teaching", aliases=("teach",), args_hint="[goal] [--mode=gui|teleop|kinesthetic] [--leader=<dev> --follower=<dev>] [--device=<dev>]"),
     CommandDef("teach stop", "Stop and distill skill", "Teaching"),
     CommandDef("teach status", "Show distillation progress", "Teaching"),
     CommandDef("teach pause", "Pause recording", "Teaching"),

src/leapflow/cli/commands/slash_handlers.py

@@ -3317,11 +3317,195 @@ async def _distill_background(session) -> None:
         _log.warning("background_distill failed", exc_info=True)
 
 
+# ---------------------------------------------------------------------------
+# Physical teach helpers (teleop / kinesthetic)
+# ---------------------------------------------------------------------------
+
+def _parse_teach_start_args(raw_args: str) -> dict[str, Any]:
+    """Parse ``--mode``, ``--leader``, ``--follower``, ``--device`` from teach start args.
+
+    Unrecognized tokens (i.e. those not prefixed with ``--``) are treated as
+    the goal string, preserving backward compatibility with the existing
+    ``/teach start <goal>`` form.
+    """
+    result: dict[str, Any] = {"mode": "gui", "goal": ""}
+    goal_parts: list[str] = []
+    tokens = raw_args.split()
+    i = 0
+    while i < len(tokens):
+        token = tokens[i]
+        if "=" in token and token.startswith("--"):
+            key, _, val = token.partition("=")
+            result[key.lstrip("-")] = val
+        elif token.startswith("--") and i + 1 < len(tokens):
+            result[token.lstrip("-")] = tokens[i + 1]
+            i += 1
+        else:
+            goal_parts.append(token)
+        i += 1
+    if goal_parts:
+        result["goal"] = " ".join(goal_parts)
+    return result
+
+
+async def _start_physical_teach(
+    ctx: "Context", mode: str, parsed: dict[str, Any]
+) -> dict[str, Any]:
+    """Start a physical teach session (teleop or kinesthetic)."""
+    from leapflow.robot.teleop import (
+        KinestheticRecorder,
+        LeaderFollowerBridge,
+        get_physical_session,
+        register_physical_session,
+    )
+
+    session_key = _physical_session_key(ctx)
+    if get_physical_session(session_key) is not None:
+        return {
+            "ok": False,
+            "message": "A physical teach session is already active. Use '/teach stop' first.",
+            "session_mode": "learning",
+        }
+
+    registry = getattr(ctx, "hardware_registry", None)
+    if registry is None:
+        return {"ok": False, "message": "Hardware subsystem is not available."}
+
+    goal = parsed.get("goal", "")
+
+    if mode == "teleop":
+        leader = parsed.get("leader", "")
+        follower = parsed.get("follower", "")
+        if not leader or not follower:
+            return {
+                "ok": False,
+                "message": "Teleop mode requires --leader=<device> --follower=<device>.",
+            }
+
+        # Build HardwareTools so the bridge writes through the governed path
+        # (SafetyPolicy, envelope, approval, trust) rather than direct transport.
+        hardware_tools = None
+        try:
+            from leapflow.hardware.tools import HardwareTools
+
+            hardware_tools = HardwareTools(
+                registry,
+                gate=getattr(ctx, "hardware_approval_gate", None)
+                or getattr(ctx, "_hardware_approval_gate", None),
+                session_id=getattr(ctx, "session_id", "") or "",
+                hardware_trust_gate=getattr(ctx, "hardware_trust_gate", None)
+                or getattr(ctx, "_hardware_trust_gate", None),
+            )
+        except Exception as exc:  # noqa: BLE001 - degrade to direct writes
+            import logging as _log
+
+            _log.getLogger(__name__).warning(
+                "Could not build HardwareTools for teleop; writes will bypass "
+                "SafetyPolicy and approval gates: %s",
+                exc,
+            )
+
+        bridge = LeaderFollowerBridge(
+            registry,
+            leader_device_id=leader,
+            follower_device_id=follower,
+            goal=goal,
+            hardware_tools=hardware_tools,
+        )
+    else:  # kinesthetic
+        device = parsed.get("device", "")
+        if not device:
+            return {
+                "ok": False,
+                "message": "Kinesthetic mode requires --device=<device>.",
+            }
+        bridge = KinestheticRecorder(registry, device_id=device, goal=goal)
+
+    await bridge.start()
+    register_physical_session(session_key, bridge)
+
+    label = "Teleop" if mode == "teleop" else "Kinesthetic"
+    msg = f"{label} teach started — session {bridge.session_id}"
+    if goal:
+        msg += f"\nGoal: {goal}"
+    msg += "\nSay '/teach stop' to stop and save the trajectory."
+    return {"ok": True, "message": msg, "session_mode": "learning"}
+
+
+async def _stop_physical_teach(ctx: "Context") -> dict[str, Any] | None:
+    """Stop an active physical teach session if one exists.  Returns None otherwise."""
+    from leapflow.robot.teleop import get_physical_session, remove_physical_session
+
+    session_key = _physical_session_key(ctx)
+    bridge = get_physical_session(session_key)
+    if bridge is None:
+        return None
+
+    trajectory = await bridge.stop()
+    remove_physical_session(session_key)
+
+    # Optional episode export: when a robot.episode_export_dir is configured,
+    # persist the trajectory as a training episode so offline learning
+    # pipelines can consume it.
+    export_info: dict[str, Any] | None = None
+    try:
+        settings = getattr(ctx, "settings", None)
+        export_dir = (
+            getattr(settings, "get", lambda *a: None)("robot.episode_export_dir")
+            if settings is not None
+            else None
+        )
+        if export_dir:
+            from leapflow.robot.episode_export import EpisodeExporter
+
+            exporter = EpisodeExporter(export_dir)
+            goal = getattr(trajectory, "goal", "") or ""
+            export_info = await exporter.export_trajectory(trajectory, task=goal)
+    except Exception as exc:  # noqa: BLE001 - export failure must not block teach stop
+        import logging
+
+        logging.getLogger(__name__).warning(
+            "Episode export failed (non-fatal): %s", exc, exc_info=True,
+        )
+
+    msg = (
+        f"Physical recording stopped — {trajectory.sample_count} samples, "
+        f"{trajectory.duration_s:.1f}s"
+    )
+    if export_info:
+        msg += f"\nEpisode exported: {export_info.get('format', '?')} "
+        msg += f"({export_info.get('step_count', 0)} steps)"
+
+    return {
+        "ok": True,
+        "message": msg,
+        "sample_count": trajectory.sample_count,
+        "duration": trajectory.duration_s,
+        "trajectory_id": trajectory.trajectory_id,
+        "export": export_info,
+        "session_mode": "idle",
+    }
+
+
+def _physical_session_key(ctx: "Context") -> str:
+    """Derive a stable key for tracking physical sessions on *ctx*."""
+    session = getattr(ctx, "session", None)
+    if session is not None:
+        sid = getattr(session, "session_id", "")
+        if sid:
+            return f"physical:{sid}"
+    return "physical:default"
+
+
 async def _execute_teach(ctx: "Context", name: str, args: str) -> dict[str, Any]:
     """Execute teach commands.
 
     Returns ``session_mode`` in the payload so the TUI client can track
     whether it should route subsequent inputs as annotations.
+
+    Physical modes (``--mode=teleop`` / ``--mode=kinesthetic``) start a
+    hardware bridge instead of the desktop GUI recorder.  The existing
+    GUI path (``--mode=gui`` or no ``--mode``) is completely unchanged.
     """
     from leapflow.engine.session.session import SessionMode
 
@@ -3331,7 +3515,16 @@ async def _execute_teach(ctx: "Context", name: str, args: str) -> dict[str, Any]
             return {"ok": False, "message": "No active session.", "session_mode": "idle"}
         if ctx.session.mode == SessionMode.LEARNING:
             return {"ok": False, "message": "Already in teaching mode. Say '/teach stop' to end.", "session_mode": "learning"}
-        goal = args if name == "teach start" else ""
+
+        # Parse --mode and physical-mode arguments from the raw args.
+        parsed = _parse_teach_start_args(args if name == "teach start" else "")
+        mode = parsed.get("mode", "gui")
+
+        if mode in ("teleop", "kinesthetic"):
+            return await _start_physical_teach(ctx, mode, parsed)
+
+        # Default GUI path — unchanged.
+        goal = parsed.get("goal", "")
         try:
             session = await ctx.session.enter_learning(goal=goal)
             msg = f"Teaching started — session {session.session_id}"
@@ -3343,6 +3536,11 @@ async def _execute_teach(ctx: "Context", name: str, args: str) -> dict[str, Any]
             return {"ok": False, "message": str(e)}
 
     if name == "teach stop":
+        # Check for an active physical session first.
+        physical_result = await _stop_physical_teach(ctx)
+        if physical_result is not None:
+            return physical_result
+
         if not ctx.session or ctx.session.mode != SessionMode.LEARNING:
             return {"ok": False, "message": "Not in teaching mode.", "session_mode": "idle"}
         try:

src/leapflow/daemon/_service_helpers.py

@@ -96,6 +96,33 @@ def engine_context_metadata(engine: Any | None, settings: Any) -> dict[str, Any]
     return metadata
 
 
+def engine_runtime_extras(engine: Any | None) -> dict[str, Any]:
+    """Return session turn count and cache hit rate for status introspection.
+
+    These live on the engine, not the budget snapshot, and are needed by the
+    self-awareness runtime facet. Kept separate from ``engine_context_metadata``
+    so they ride only on the (rare) status call, never on every stream chunk.
+    The cache hit rate is normalised from the tracker's 0..1 fraction to a
+    human-facing percentage.
+    """
+    extras: dict[str, Any] = {}
+    if engine is None:
+        return extras
+    turn_count = getattr(engine, "_session_turn_count", None)
+    if isinstance(turn_count, int) and turn_count >= 0:
+        extras["session_turn_count"] = turn_count
+    tracker = getattr(engine, "_usage_tracker", None)
+    summary_fn = getattr(tracker, "summary", None) if tracker is not None else None
+    if callable(summary_fn):
+        try:
+            rate = getattr(summary_fn(), "cache_hit_rate", None)
+        except Exception:  # noqa: BLE001 - telemetry must never fail status
+            rate = None
+        if isinstance(rate, (int, float)) and rate >= 0:
+            extras["cache_hit_rate"] = round(float(rate) * 100, 1)
+    return extras
+
+
 def host_backend_status(ctx: Any | None) -> dict[str, Any]:
     """Inspect daemon host-backend state for status reporting."""
     if ctx is None: