feat(agents): Codex gets a model list, an effort and images (A3c of spec 30)

README.md

@@ -91,8 +91,8 @@ itself.
 Claude Code is the full experience. Antigravity and Codex are off until you list them in Botloft's
 `config.toml` (`experimental_agents = ["agy"]`, `["codex"]` or both); then a new bot can be made on
 them, and Settings has "Agent for new bots". A Codex bot asks you before it changes a file or runs a
-command, like Claude Code, and shows the exact size of its conversation; it can still read any file,
-and it has no model picker or images yet. See [spec section 30](docs/spec.md) for what was
+command, like Claude Code, and shows the exact size of its conversation; it can still read any file.
+It has its own model list and effort picker, and understands images. See [spec section 30](docs/spec.md) for what was
 checked.
 
 ## Get started

app/src/features/chat/ChatComposer.tsx

@@ -253,8 +253,13 @@ export const ChatComposer = memo(function ChatComposer({
                 <EffortPicker bot={bot} onLater={setLater} />
                 <ModelPicker bot={bot} onLater={setLater} />
               </>
+            ) : bot.agent === "codex" ? (
+              <>
+                <EffortPicker bot={bot} onLater={setLater} />
+                <AgentModelPicker bot={bot} onLater={setLater} />
+              </>
             ) : (
-              bot.agent === "agy" && <AgentModelPicker bot={bot} onLater={setLater} />
+              <AgentModelPicker bot={bot} onLater={setLater} />
             )}
             <button
               type="submit"

app/src/features/chat/agentBot.test.tsx

@@ -88,3 +88,20 @@ describe("Claude Code missing", () => {
     expect(await screen.findByText("Bots can't start")).toBeDefined();
   });
 });
+
+describe("a Codex bot", () => {
+  test("has an effort and a model list under the message box", async () => {
+    const fake = new FakeBotloft();
+    const crew = fake.addCrew("Ops");
+    const bot = fake.addBot(crew.id, "Cody");
+    fake.bot(bot.id).agent = "codex";
+    fake.setBotState(bot.id, "idle");
+    renderApp(fake);
+    await crewOpened("Ops");
+    openBot("Cody");
+    await screen.findByRole("list", { name: "Messages" });
+    expect(screen.getByRole("button", { name: /^Effort:/ })).toBeDefined();
+    fireEvent.click(screen.getByRole("button", { name: "Model: Default" }));
+    expect(await screen.findByRole("menu", { name: "Model" })).toBeDefined();
+  });
+});

app/src/i18n/en/bots.ts

@@ -117,7 +117,7 @@ export const bots = {
       experimental:
         "Experimental. This bot cannot ask you before it acts. It can work with files in its own folders and use the crew's tools, but it cannot run commands, and it has no usage meter or model choice.",
       experimentalCodex:
-        "Experimental. This bot asks you before it changes a file or runs a command, like Claude Code. It can still read any file, and it has no model or image choice yet.",
+        "Experimental. This bot asks you before it changes a file or runs a command, like Claude Code. It can still read any file.",
     },
     color: "Color",
     swatch: (color: string) => `Color ${color}`,

app/src/i18n/es/bots.ts

@@ -115,7 +115,7 @@ export const bots: Messages["bots"] = {
       experimental:
         "Experimental. Este bot no puede preguntarte antes de actuar. Trabaja con archivos de sus propias carpetas y usa las herramientas del equipo, pero no ejecuta comandos, y no tiene medidor de uso ni elección de modelo.",
       experimentalCodex:
-        "Experimental. Este bot te pregunta antes de cambiar un archivo o ejecutar un comando, como Claude Code. Todavía puede leer cualquier archivo, y no tiene elección de modelo ni de imagen aún.",
+        "Experimental. Este bot te pregunta antes de cambiar un archivo o ejecutar un comando, como Claude Code. Todavía puede leer cualquier archivo.",
     },
     color: "Color",
     swatch: (color: string) => `Color ${color}`,

app/src/i18n/pt-BR/bots.ts

@@ -114,7 +114,7 @@ export const bots: Messages["bots"] = {
       experimental:
         "Experimental. Este bot não consegue perguntar a você antes de agir. Ele trabalha com arquivos das próprias pastas e usa as ferramentas da equipe, mas não roda comandos, e não tem medidor de uso nem escolha de modelo.",
       experimentalCodex:
-        "Experimental. Este bot pergunta a você antes de mudar um arquivo ou rodar um comando, como o Claude Code. Ele ainda pode ler qualquer arquivo, e não tem escolha de modelo nem de imagem ainda.",
+        "Experimental. Este bot pergunta a você antes de mudar um arquivo ou rodar um comando, como o Claude Code. Ele ainda pode ler qualquer arquivo.",
     },
     color: "Cor",
     swatch: (color: string) => `Cor ${color}`,

crates/botloftd/src/agent/codex.rs

@@ -11,7 +11,7 @@ use std::sync::Arc;
 use botloft_core::ids::BotId;
 use botloft_core::protocol::AgentKind;
 use bytes::Bytes;
-use serde_json::Value;
+use serde_json::{Value, json};
 
 use super::codex_decoder::CodexDecoder;
 use super::codex_wire::{CodexControl, Shared};
@@ -101,7 +101,7 @@ impl Agent for CodexAgent {
 
     fn encode_turn(&self, _uuid: &str, turn: &Turn) -> Bytes {
         // The wrapper turns this neutral line into a `turn/start`.
-        super::neutral_turn(turn)
+        super::neutral_turn_with_images(turn)
     }
 
     fn decoder(&self) -> Box<dyn OutputDecoder> {
@@ -159,3 +159,114 @@ fn owner_mcp_servers() -> Vec<String> {
         })
         .unwrap_or_default()
 }
+
+/// The models `codex app-server` lists (`model/list`): a short-lived server
+/// of its own, asked once. The owner's apps and plugins stay off, as for a
+/// bot.
+pub fn models(program: &Path) -> io::Result<Vec<botloft_core::protocol::AgentModel>> {
+    use std::io::{BufRead, BufReader, Write};
+    use std::process::{Command, Stdio};
+    use std::sync::mpsc;
+    use std::time::Duration;
+
+    let mut child = Command::new(program)
+        .args([
+            "app-server",
+            "-c",
+            "features.apps=false",
+            "-c",
+            "features.plugins=false",
+        ])
+        .stdin(Stdio::piped())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::null())
+        .spawn()?;
+    let mut stdin = child
+        .stdin
+        .take()
+        .ok_or_else(|| io::Error::other("no stdin"))?;
+    let stdout = child
+        .stdout
+        .take()
+        .ok_or_else(|| io::Error::other("no stdout"))?;
+    let (lines, answers) = mpsc::channel::<String>();
+    std::thread::spawn(move || {
+        for line in BufReader::new(stdout).lines().map_while(Result::ok) {
+            if lines.send(line).is_err() {
+                break;
+            }
+        }
+    });
+    let send = |stdin: &mut std::process::ChildStdin, message: Value| {
+        let mut line = message.to_string();
+        line.push('\n');
+        stdin.write_all(line.as_bytes())
+    };
+    let ask = |stdin: &mut std::process::ChildStdin, id: u64, method: &str, params: Value| {
+        send(
+            stdin,
+            json!({ "id": id, "method": method, "params": params }),
+        )?;
+        loop {
+            let line = answers
+                .recv_timeout(Duration::from_secs(20))
+                .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "codex did not answer"))?;
+            let Ok(value) = serde_json::from_str::<Value>(&line) else {
+                continue;
+            };
+            if value["id"] == id {
+                return Ok(value);
+            }
+        }
+    };
+    let outcome = (|| -> io::Result<Value> {
+        ask(
+            &mut stdin,
+            1,
+            "initialize",
+            json!({ "clientInfo": { "name": "botloft", "version": env!("CARGO_PKG_VERSION") } }),
+        )?;
+        send(&mut stdin, json!({ "method": "initialized", "params": {} }))?;
+        ask(&mut stdin, 2, "model/list", json!({}))
+    })();
+    let _ = child.kill();
+    let _ = child.wait();
+    Ok(parse_models(&outcome?["result"]))
+}
+
+fn parse_models(result: &Value) -> Vec<botloft_core::protocol::AgentModel> {
+    result["data"]
+        .as_array()
+        .map(|models| {
+            models
+                .iter()
+                .filter(|model| model["hidden"].as_bool() != Some(true))
+                .filter_map(|model| {
+                    let id = model["model"].as_str().or_else(|| model["id"].as_str())?;
+                    Some(botloft_core::protocol::AgentModel {
+                        id: id.to_owned(),
+                        name: model["displayName"].as_str().unwrap_or(id).to_owned(),
+                    })
+                })
+                .collect()
+        })
+        .unwrap_or_default()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn the_list_skips_hidden_models_and_names_the_rest() {
+        let models = parse_models(&json!({ "data": [
+            { "id": "a", "model": "gpt-a", "displayName": "GPT A", "hidden": false },
+            { "id": "b", "model": "gpt-b", "displayName": "GPT B", "hidden": true },
+            { "id": "c", "model": "gpt-c" },
+        ] }));
+        let ids: Vec<&str> = models.iter().map(|m| m.id.as_str()).collect();
+        assert_eq!(ids, ["gpt-a", "gpt-c"]);
+        assert_eq!(models[0].name, "GPT A");
+        assert_eq!(models[1].name, "gpt-c");
+    }
+}

crates/botloftd/src/agent/codex_wire.rs

@@ -39,8 +39,8 @@ pub(super) struct State {
     next_id: u64,
     phase: Phase,
     pending: HashMap<u64, Pending>,
-    /// Owner messages (text) waiting for the thread or for the running turn.
-    queue: VecDeque<String>,
+    /// The input of messages waiting for the thread or for the running turn.
+    queue: VecDeque<Vec<Value>>,
     turn_active: bool,
 }
 
@@ -125,12 +125,29 @@ impl Shared {
     pub(super) fn enqueue(&self, line: &[u8]) -> io::Result<()> {
         let event: Value = serde_json::from_slice(line)
             .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?;
-        let text = event["message"]["content"][0]["text"]
-            .as_str()
-            .unwrap_or_default()
-            .to_owned();
+        // Text as it is; an image as the data URL `turn/start` takes.
+        let input: Vec<Value> = event["message"]["content"]
+            .as_array()
+            .map(|blocks| {
+                blocks
+                    .iter()
+                    .filter_map(|block| match block["type"].as_str() {
+                        Some("text") => Some(json!({ "type": "text", "text": block["text"] })),
+                        Some("image") => Some(json!({
+                            "type": "image",
+                            "url": format!(
+                                "data:{};base64,{}",
+                                block["media_type"].as_str().unwrap_or("image/png"),
+                                block["data"].as_str().unwrap_or_default()
+                            ),
+                        })),
+                        _ => None,
+                    })
+                    .collect()
+            })
+            .unwrap_or_default();
         let mut state = self.lock();
-        state.queue.push_back(text);
+        state.queue.push_back(input);
         self.pump(&mut state);
         Ok(())
     }
@@ -142,12 +159,12 @@ impl Shared {
         if state.turn_active {
             return;
         }
-        let Some(text) = state.queue.pop_front() else {
+        let Some(input) = state.queue.pop_front() else {
             return;
         };
         let mut params = json!({
             "threadId": thread,
-            "input": [{ "type": "text", "text": text }],
+            "input": input,
         });
         if let Some(effort) = &self.input.effort {
             params["effort"] = json!(effort);
@@ -383,4 +400,32 @@ mod tests {
         let last = written.lock().expect("lock").last().cloned().expect("line");
         assert_eq!(last["method"], "thread/start");
     }
+
+    #[test]
+    fn an_image_goes_as_a_data_url_with_the_text() {
+        let sink = Sink::default();
+        let written = Arc::clone(&sink.0);
+        let shared = Shared::begin(Box::new(sink), input(None));
+        shared.answered(&json!({ "id": 1, "result": {} }));
+        shared.answered(&json!({ "id": 2, "result": { "thread": { "id": "th" } } }));
+        let line = json!({ "event": "user", "message": { "role": "user", "content": [
+            { "type": "text", "text": "look" },
+            { "type": "image", "media_type": "image/png", "data": "AAAA" },
+        ] } })
+        .to_string();
+        shared.enqueue(line.as_bytes()).expect("queued");
+        let turn = written
+            .lock()
+            .expect("lock")
+            .iter()
+            .find(|line| line["method"] == "turn/start")
+            .cloned()
+            .expect("turn");
+        assert_eq!(turn["params"]["input"][0]["text"], "look");
+        assert_eq!(turn["params"]["input"][1]["type"], "image");
+        assert_eq!(
+            turn["params"]["input"][1]["url"],
+            "data:image/png;base64,AAAA"
+        );
+    }
 }

crates/botloftd/src/agent/mod.rs

@@ -21,7 +21,7 @@ use serde_json::Value;
 
 pub use agy::{AgyAgent, models as agy_models};
 pub use claude::ClaudeAgent;
-pub use codex::CodexAgent;
+pub use codex::{CodexAgent, models as codex_models};
 
 use crate::runtime::ProcessControl;
 use crate::state::Daemon;
@@ -145,12 +145,21 @@ pub trait Agent: Send + Sync + 'static {
     }
 }
 
-/// Botloft's own line for a turn, for agents whose stdin is a protocol of
-/// their own that a wrapper speaks: text only, as `agy` takes it.
-pub(crate) fn neutral_turn(turn: &Turn) -> Bytes {
+/// Botloft's own line for a turn, for an agent whose stdin is a protocol of
+/// its own that a wrapper speaks (Codex): the text, then each image as a block
+/// with its media type and base64 data.
+pub(crate) fn neutral_turn_with_images(turn: &Turn) -> Bytes {
+    let mut content = vec![serde_json::json!({ "type": "text", "text": turn.text })];
+    content.extend(turn.images.iter().map(|image| {
+        serde_json::json!({
+            "type": "image",
+            "media_type": image.media_type,
+            "data": image.data,
+        })
+    }));
     let event = serde_json::json!({
         "event": "user",
-        "message": { "role": "user", "content": [{ "type": "text", "text": turn.text }] },
+        "message": { "role": "user", "content": content },
     });
     let mut line = event.to_string().into_bytes();
     line.push(b'\n');

crates/botloftd/src/service/models.rs

@@ -144,32 +144,41 @@ pub fn set_agent_model(daemon: &Daemon, params: BotsSetAgentModelParams) -> ApiR
     Ok(bot)
 }
 
-/// The models the agent offers, as it lists them. Kept for a while: asking
-/// takes seconds.
+/// The models the agent offers, as it lists them. Kept for a while, for each
+/// agent: asking takes seconds.
 pub fn agent_models(daemon: &Daemon, params: AgentsModelsParams) -> ApiResult<Vec<AgentModel>> {
     use std::sync::Mutex;
     use std::time::{Duration, Instant};
-    static CACHE: Mutex<Option<(Instant, Vec<AgentModel>)>> = Mutex::new(None);
-    if params.agent != AgentKind::Agy {
-        return Err(ApiError::validation(
-            "only agy lists its models; Claude Code's are fixed",
-        ));
-    }
+    type Cached = Option<(Instant, Vec<AgentModel>)>;
+    static CACHE: Mutex<[Cached; 2]> = Mutex::new([None, None]);
+    let slot = match params.agent {
+        AgentKind::Agy => 0,
+        AgentKind::Codex => 1,
+        AgentKind::Claude => {
+            return Err(ApiError::validation(
+                "Claude Code's models are fixed; only the other agents list theirs",
+            ));
+        }
+    };
     let mut cache = CACHE
         .lock()
         .unwrap_or_else(|poisoned| poisoned.into_inner());
-    if let Some((at, models)) = cache.as_ref()
+    if let Some((at, models)) = cache[slot].as_ref()
         && at.elapsed() < Duration::from_secs(600)
     {
         return Ok(models.clone());
     }
-    let agent = crate::agent::of(AgentKind::Agy)
-        .ok_or_else(|| ApiError::validation("agy is not available"))?;
+    let agent = crate::agent::of(params.agent)
+        .ok_or_else(|| ApiError::validation("that agent is not available"))?;
     let program = agent
-        .locate(daemon.supervisor.agy_path(), None)
+        .locate(daemon.supervisor.program_setting(params.agent), None)
         .map_err(|err| ApiError::Conflict(err.to_string()))?;
-    let models = crate::agent::agy_models(&program)
-        .map_err(|err| ApiError::Conflict(format!("could not list the models: {err}")))?;
-    *cache = Some((Instant::now(), models.clone()));
+    let listed = match params.agent {
+        AgentKind::Codex => crate::agent::codex_models(&program),
+        _ => crate::agent::agy_models(&program),
+    };
+    let models =
+        listed.map_err(|err| ApiError::Conflict(format!("could not list the models: {err}")))?;
+    cache[slot] = Some((Instant::now(), models.clone()));
     Ok(models)
 }