feat: integrate LogAnalysisBot for enhanced failure analysis in auto_memory

src/microbots/auto_memory/analyzer.py

@@ -1,104 +1,209 @@
-"""Failure analysis utilities that convert callback results into structured feedback."""
+"""Failure analysis that uses LogAnalysisBot to turn callback failures into feedback."""
 
 from __future__ import annotations
 
+import tempfile
+from logging import getLogger
 from pathlib import Path
 
 from microbots.auto_memory.callbacks import CallbackResult
 from microbots.auto_memory.data_models import Feedback
+from microbots.bot.LogAnalysisBot import LogAnalysisBot
 
-# Maximum bytes read from a log tail when building root-cause snippets.
-_MAX_LOG_BYTES = 4096
+logger = getLogger(__name__)
+
+# Per-stream tail cap when assembling the combined log fed to the LLM
+# analyzer. Prevents pathological callback outputs (e.g. verbose pytest
+# tracebacks or dumped fixtures) from blowing up model context / cost.
+_LOG_TAIL_BYTES = 16 * 1024
 
 
 def analyze_failure(
     callback_results: list[CallbackResult],
     candidate_path: Path,
     iteration_idx: int,
+    *,
+    analyzer_model: str,
+    analyzer_max_iterations: int,
+    analyzer_timeout_s: int,
 ) -> Feedback:
-    """Produce a :class:`~microbots.auto_memory.data_models.Feedback` from callback results.
+    """Produce a :class:`~microbots.auto_memory.data_models.Feedback` via LogAnalysisBot.
+
+    Combines the stdout/stderr of every failed callback into a single log
+    file and hands it to a fresh
+    :class:`~microbots.bot.LogAnalysisBot.LogAnalysisBot` with the candidate
+    directory mounted read-only. The bot's final narrative answer is stored
+    as :attr:`Feedback.summary`; :attr:`Feedback.root_causes` is left empty
+    on the success path because the LLM emits a single diagnosis rather
+    than a discrete list.
 
-    Inspects every :class:`~microbots.auto_memory.callbacks.CallbackResult`
-    and assembles a structured failure report that the agent can read on
-    the next iteration.
+    On no failures, returns a minimal ``All callbacks passed.`` feedback
+    without invoking the bot. If the bot raises or fails to produce a
+    result, :attr:`Feedback.summary` falls back to a short
+    ``"<N> of <M> callback(s) failed: ..."`` string and
+    :attr:`Feedback.root_causes` contains the analyzer error.
+
+    In every non-passing case :attr:`Feedback.validator_failures` lists the
+    names of the callbacks that did not pass.
 
     Parameters
     ----------
     callback_results : list[CallbackResult]
         Results returned by
         :meth:`~microbots.auto_memory.callbacks.ShellCallbackRunner.run_all`.
     candidate_path : Path
-        Path to the candidate output for this iteration.  Reserved for
-        future use (e.g. static analysis); not read directly at present.
+        Path to the candidate output for this iteration; mounted read-only
+        into the bot so it can correlate log entries with source.
     iteration_idx : int
         Zero-based index of the iteration that produced these results.
+    analyzer_model : str
+        LiteLLM model identifier for :class:`~microbots.bot.LogAnalysisBot.LogAnalysisBot`.
+    analyzer_max_iterations : int
+        Maximum bot iterations to run.
+    analyzer_timeout_s : int
+        Bot timeout in seconds.
 
     Returns
     -------
     Feedback
         Structured summary with ``validator_failures``, ``root_causes``,
-        ``suggested_actions``, and a human-readable ``summary`` string.
+        and a human-readable ``summary`` string.
     """
     failed = [r for r in callback_results if not r.passed]
-
     validator_failures: list[str] = [r.spec.name for r in failed]
-    root_causes: list[str] = []
-    suggested_actions: list[str] = []
 
-    for result in failed:
-        if result.timed_out:
-            root_causes.append(
-                f"{result.spec.name}: timed out after {result.spec.timeout_s}s"
-            )
-            suggested_actions.append(
-                f"Reduce runtime of {result.spec.name!r} or increase its timeout."
-            )
-        else:
-            log_snippet = _read_tail(result.stderr_path) or _read_tail(result.stdout_path)
-            if log_snippet:
-                root_causes.append(f"{result.spec.name}: {log_snippet[:200]}")
-            else:
-                root_causes.append(
-                    f"{result.spec.name}: exited with code {result.return_code}"
-                    f" (expected {result.spec.expected_return_code})"
-                )
-
-    if failed:
-        names = ", ".join(validator_failures)
-        summary = f"{len(failed)} of {len(callback_results)} callback(s) failed: {names}"
-    else:
-        summary = "All callbacks passed."
+    if not failed:
+        return Feedback(iteration_idx=iteration_idx, summary="All callbacks passed.")
+
+    summary = (
+        f"{len(failed)} of {len(callback_results)} callback(s) failed: "
+        f"{', '.join(validator_failures)}"
+    )
+
+    # LogAnalysisBot needs a directory for its read-only mount.
+    mount_folder = (
+        candidate_path
+        if candidate_path.exists() and candidate_path.is_dir()
+        else candidate_path.parent
+    )
+
+    combined_log = _write_combined_log(failed)
+    try:
+        bot = LogAnalysisBot(
+            model=analyzer_model,
+            folder_to_mount=str(mount_folder),
+        )
+        bot_result = bot.run(
+            file_name=str(combined_log),
+            max_iterations=analyzer_max_iterations,
+            timeout_in_seconds=analyzer_timeout_s,
+        )
+    except Exception as exc:
+        logger.warning("LogAnalysisBot invocation failed: %s", exc, exc_info=True)
+        return Feedback(
+            iteration_idx=iteration_idx,
+            summary=summary,
+            validator_failures=validator_failures,
+            root_causes=[f"LLM analyzer error: {exc}"],
+        )
+    finally:
+        combined_log.unlink(missing_ok=True)
+
+    result = (bot_result.result or "").strip()
+    if bot_result.status and result:
+        # The LLM produces one narrative diagnosis, not a list of causes;
+        # store it as the summary and leave root_causes empty.
+        return Feedback(
+            iteration_idx=iteration_idx,
+            summary=result,
+            validator_failures=validator_failures,
+        )
 
     return Feedback(
         iteration_idx=iteration_idx,
         summary=summary,
-        root_causes=root_causes,
         validator_failures=validator_failures,
-        suggested_actions=suggested_actions,
+        root_causes=[
+            f"LLM analyzer did not produce a result: {bot_result.error or 'unknown'}"
+        ],
     )
 
 
-def _read_tail(path: Path, max_bytes: int = _MAX_LOG_BYTES) -> str:
-    """Return the last *max_bytes* of *path* as a stripped string, or ``''`` if missing.
+def _write_combined_log(failed: list[CallbackResult]) -> Path:
+    """Write a temp log file combining every failed callback's stdout/stderr.
+
+    Parameters
+    ----------
+    failed : list[CallbackResult]
+        Results of the callbacks that did not pass.
+
+    Returns
+    -------
+    Path
+        Path to the newly created combined log file. Caller is responsible
+        for deleting it.
+    """
+    with tempfile.NamedTemporaryFile(
+        mode="w", suffix=".log", delete=False, encoding="utf-8"
+    ) as fh:
+        for r in failed:
+            fh.write(f"\n===== callback: {r.spec.name} =====\n")
+            fh.write(
+                f"return_code={r.return_code} "
+                f"expected={r.spec.expected_return_code} "
+                f"timed_out={r.timed_out} "
+                f"duration_s={r.duration_s:.2f}\n"
+            )
+            fh.write("--- stderr ---\n")
+            fh.write(_safe_read(r.stderr_path))
+            fh.write("\n--- stdout ---\n")
+            fh.write(_safe_read(r.stdout_path))
+            fh.write("\n")
+        return Path(fh.name)
+
+
+def _safe_read(path: Path, max_bytes: int = _LOG_TAIL_BYTES) -> str:
+    """Return the tail of *path*'s text content, or a placeholder on I/O error.
+
+    Only the last ``max_bytes`` bytes are read to bound memory usage and the
+    downstream LLM analyzer context. When the file exceeds that cap a
+    ``"<truncated: showing last N bytes of M>\\n"`` marker is prepended so
+    the model knows the view is partial.
 
     Parameters
     ----------
     path : Path
-        File to read from.
+        Filesystem path to read.
     max_bytes : int, optional
-        Maximum number of bytes to read from the end of the file.
+        Maximum number of trailing bytes to return. Defaults to
+        ``_LOG_TAIL_BYTES``. A value ``<= 0`` disables the cap.
 
     Returns
     -------
     str
-        Decoded and stripped tail of the file, or an empty string if the
-        file cannot be opened.
+        The (possibly truncated) file contents decoded as UTF-8 (invalid
+        bytes replaced), or ``"<log unavailable>"`` if the file cannot be
+        read.
     """
     try:
-        with path.open("rb") as fh:
-            fh.seek(0, 2)
-            size = fh.tell()
-            fh.seek(max(0, size - max_bytes))
-            return fh.read().decode("utf-8", errors="replace").strip()
+        if max_bytes <= 0:
+            data = path.read_bytes()
+            truncated = False
+            total = len(data)
+        else:
+            size = path.stat().st_size
+            with path.open("rb") as fh:
+                if size > max_bytes:
+                    fh.seek(size - max_bytes)
+                    truncated = True
+                else:
+                    truncated = False
+                data = fh.read()
+            total = size
     except OSError:
-        return ""
+        return "<log unavailable>"
+
+    text = data.decode("utf-8", errors="replace")
+    if truncated:
+        return f"<truncated: showing last {len(data)} bytes of {total}>\n{text}"
+    return text

src/microbots/auto_memory/cli.py

@@ -50,7 +50,7 @@ def run_from_yaml(
         generated to avoid collisions.
     model : str
         Model identifier forwarded to :class:`WritingBotRunner` (required,
-        keyword-only — e.g. ``"azure/gpt-4o"``).
+        keyword-only — e.g. ``"azure-openai/gpt-4o"``).
 
     Returns
     -------

src/microbots/auto_memory/config.py

@@ -9,6 +9,7 @@
 
 from microbots.auto_memory.errors import ConfigError
 from microbots.auto_memory.data_models import CallbackSpec, ReferenceInput
+from microbots.constants import ModelProvider
 
 logger = getLogger(__name__)
 
@@ -30,6 +31,11 @@ class TaskConfig:
     timeout_min: int = 60
     per_iteration_timeout: int = 600    # seconds
 
+    # --- analyzer (LogAnalysisBot) settings ---
+    analyzer_model: str = "azure-openai/gpt-4o"
+    analyzer_max_iterations: int = 20
+    analyzer_timeout_s: int = 300
+
     # -----------------------------------------------------------------------
 
     @classmethod
@@ -103,6 +109,9 @@ def load_from_yaml(cls, path: str) -> "TaskConfig":
             max_iterations=int(data.get("max_iterations", 5)),
             timeout_min=int(data.get("timeout_min", 60)),
             per_iteration_timeout=int(data.get("per_iteration_timeout", 600)),
+            analyzer_model=str(data.get("analyzer_model", "azure-openai/gpt-4o")),
+            analyzer_max_iterations=int(data.get("analyzer_max_iterations", 20)),
+            analyzer_timeout_s=int(data.get("analyzer_timeout_s", 300)),
         )
         config.validate()
         return config
@@ -130,6 +139,35 @@ def validate(self) -> None:
                 f"'per_iteration_timeout' must be >= 1, got {self.per_iteration_timeout}"
             )
 
+        if not self.analyzer_model:
+            raise ConfigError("'analyzer_model' must not be empty")
+
+        # Mirror MicroBot._validate_model_and_provider so we fail fast at
+        # config load time instead of deferring to a runtime ValueError
+        # when LogAnalysisBot is instantiated.
+        if self.analyzer_model.count("/") != 1:
+            raise ConfigError(
+                f"'analyzer_model' must be in the format '<provider>/<model_name>', "
+                f"got '{self.analyzer_model}'"
+            )
+        provider = self.analyzer_model.split("/", 1)[0]
+        supported = [e.value for e in ModelProvider]
+        if provider not in supported:
+            raise ConfigError(
+                f"'analyzer_model' has unsupported provider '{provider}'; "
+                f"expected one of {supported}"
+            )
+
+        if self.analyzer_max_iterations < 1:
+            raise ConfigError(
+                f"'analyzer_max_iterations' must be >= 1, got {self.analyzer_max_iterations}"
+            )
+
+        if self.analyzer_timeout_s < 1:
+            raise ConfigError(
+                f"'analyzer_timeout_s' must be >= 1, got {self.analyzer_timeout_s}"
+            )
+
         if self.output_format not in ("file", "dir", "stdout"):
             raise ConfigError(
                 f"'output_format' must be 'file', 'dir', or 'stdout', got '{self.output_format}'"

src/microbots/auto_memory/orchestrator.py

@@ -71,7 +71,7 @@ class TrainingLoopOrchestrator:
 
         orchestrator = TrainingLoopOrchestrator(
             config=cfg,
-            agent_runner=WritingBotRunner(model="azure/gpt-4o"),
+            agent_runner=WritingBotRunner(model="azure-openai/gpt-4o"),
             callback_runner=ShellCallbackRunner(),
             workspace=WorkspaceManager(run_dir=Path("runs/my_task")),
         )
@@ -380,4 +380,7 @@ def analyze_failure(
             callback_results=callback_results,
             candidate_path=candidate_path,
             iteration_idx=iteration_idx,
+            analyzer_model=self._config.analyzer_model,
+            analyzer_max_iterations=self._config.analyzer_max_iterations,
+            analyzer_timeout_s=self._config.analyzer_timeout_s,
         )

src/microbots/auto_memory/runners/writing_bot_runner.py

@@ -24,7 +24,7 @@ class WritingBotRunner:
     ----------
     model : str
         Model identifier forwarded to ``WritingBot`` (e.g.
-        ``"azure/gpt-4o"``).
+        ``"azure-openai/gpt-4o"``).
     max_iterations : int, optional
         Maximum number of agent steps per run; forwarded to
         ``WritingBot.run()``. Defaults to 20.
@@ -36,7 +36,7 @@ def __init__(self, model: str, max_iterations: int = 20) -> None:
         Parameters
         ----------
         model : str
-            Model identifier forwarded to ``WritingBot`` (e.g. ``"azure/gpt-4o"``).
+            Model identifier forwarded to ``WritingBot`` (e.g. ``"azure-openai/gpt-4o"``).
         max_iterations : int, optional
             Maximum number of agent steps per run. Defaults to 20.
         """

test/auto_memory/runners/test_writing_bot_runner.py

@@ -13,7 +13,7 @@
 # Helpers
 # ---------------------------------------------------------------------------
 
-_MODEL = "azure/gpt-4o"
+_MODEL = "azure-openai/gpt-4o"
 _TASK = "Write a summary to /memories/summary.md"