Skip to content

Commit efc9336

Browse files
authored
Merge pull request #251 from httpsphl/feat/codex-models-images
feat(agents): Codex gets a model list, an effort and images (A3c of spec 30)
2 parents 99844b6 + 55b8134 commit efc9336

11 files changed

Lines changed: 235 additions & 39 deletions

File tree

‎README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,8 +120,8 @@ Claude Code and Antigravity compare like this:
120120
Claude Code is the full experience. Antigravity and Codex are off until you list them in Botloft's
121121
`config.toml` (`experimental_agents = ["agy"]`, `["codex"]` or both); then a new bot can be made on
122122
them, and Settings has "Agent for new bots". A Codex bot asks you before it changes a file or runs a
123-
command, like Claude Code, and shows the exact size of its conversation; it can still read any file,
124-
and it has no model picker or images yet. See [spec section 30](docs/spec.md) for what was
123+
command, like Claude Code, and shows the exact size of its conversation; it can still read any file.
124+
It has its own model list and effort picker, and understands images. See [spec section 30](docs/spec.md) for what was
125125
checked.
126126

127127
## Get started

‎app/src/features/chat/ChatComposer.tsx‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -253,8 +253,13 @@ export const ChatComposer = memo(function ChatComposer({
253253
<EffortPicker bot={bot} onLater={setLater} />
254254
<ModelPicker bot={bot} onLater={setLater} />
255255
</>
256+
) : bot.agent === "codex" ? (
257+
<>
258+
<EffortPicker bot={bot} onLater={setLater} />
259+
<AgentModelPicker bot={bot} onLater={setLater} />
260+
</>
256261
) : (
257-
bot.agent === "agy" && <AgentModelPicker bot={bot} onLater={setLater} />
262+
<AgentModelPicker bot={bot} onLater={setLater} />
258263
)}
259264
<button
260265
type="submit"

‎app/src/features/chat/agentBot.test.tsx‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,3 +88,20 @@ describe("Claude Code missing", () => {
8888
expect(await screen.findByText("Bots can't start")).toBeDefined();
8989
});
9090
});
91+
92+
describe("a Codex bot", () => {
93+
test("has an effort and a model list under the message box", async () => {
94+
const fake = new FakeBotloft();
95+
const crew = fake.addCrew("Ops");
96+
const bot = fake.addBot(crew.id, "Cody");
97+
fake.bot(bot.id).agent = "codex";
98+
fake.setBotState(bot.id, "idle");
99+
renderApp(fake);
100+
await crewOpened("Ops");
101+
openBot("Cody");
102+
await screen.findByRole("list", { name: "Messages" });
103+
expect(screen.getByRole("button", { name: /^Effort:/ })).toBeDefined();
104+
fireEvent.click(screen.getByRole("button", { name: "Model: Default" }));
105+
expect(await screen.findByRole("menu", { name: "Model" })).toBeDefined();
106+
});
107+
});

‎app/src/i18n/en/bots.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ export const bots = {
117117
experimental:
118118
"Experimental. This bot cannot ask you before it acts. It can work with files in its own folders and use the crew's tools, but it cannot run commands, and it has no usage meter or model choice.",
119119
experimentalCodex:
120-
"Experimental. This bot asks you before it changes a file or runs a command, like Claude Code. It can still read any file, and it has no model or image choice yet.",
120+
"Experimental. This bot asks you before it changes a file or runs a command, like Claude Code. It can still read any file.",
121121
},
122122
color: "Color",
123123
swatch: (color: string) => `Color ${color}`,

‎app/src/i18n/es/bots.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ export const bots: Messages["bots"] = {
115115
experimental:
116116
"Experimental. Este bot no puede preguntarte antes de actuar. Trabaja con archivos de sus propias carpetas y usa las herramientas del equipo, pero no ejecuta comandos, y no tiene medidor de uso ni elección de modelo.",
117117
experimentalCodex:
118-
"Experimental. Este bot te pregunta antes de cambiar un archivo o ejecutar un comando, como Claude Code. Todavía puede leer cualquier archivo, y no tiene elección de modelo ni de imagen aún.",
118+
"Experimental. Este bot te pregunta antes de cambiar un archivo o ejecutar un comando, como Claude Code. Todavía puede leer cualquier archivo.",
119119
},
120120
color: "Color",
121121
swatch: (color: string) => `Color ${color}`,

‎app/src/i18n/pt-BR/bots.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ export const bots: Messages["bots"] = {
114114
experimental:
115115
"Experimental. Este bot não consegue perguntar a você antes de agir. Ele trabalha com arquivos das próprias pastas e usa as ferramentas da equipe, mas não roda comandos, e não tem medidor de uso nem escolha de modelo.",
116116
experimentalCodex:
117-
"Experimental. Este bot pergunta a você antes de mudar um arquivo ou rodar um comando, como o Claude Code. Ele ainda pode ler qualquer arquivo, e não tem escolha de modelo nem de imagem ainda.",
117+
"Experimental. Este bot pergunta a você antes de mudar um arquivo ou rodar um comando, como o Claude Code. Ele ainda pode ler qualquer arquivo.",
118118
},
119119
color: "Cor",
120120
swatch: (color: string) => `Cor ${color}`,

‎crates/botloftd/src/agent/codex.rs‎

Lines changed: 113 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ use std::sync::Arc;
1111
use botloft_core::ids::BotId;
1212
use botloft_core::protocol::AgentKind;
1313
use bytes::Bytes;
14-
use serde_json::Value;
14+
use serde_json::{Value, json};
1515

1616
use super::codex_decoder::CodexDecoder;
1717
use super::codex_wire::{CodexControl, Shared};
@@ -101,7 +101,7 @@ impl Agent for CodexAgent {
101101

102102
fn encode_turn(&self, _uuid: &str, turn: &Turn) -> Bytes {
103103
// The wrapper turns this neutral line into a `turn/start`.
104-
super::neutral_turn(turn)
104+
super::neutral_turn_with_images(turn)
105105
}
106106

107107
fn decoder(&self) -> Box<dyn OutputDecoder> {
@@ -159,3 +159,114 @@ fn owner_mcp_servers() -> Vec<String> {
159159
})
160160
.unwrap_or_default()
161161
}
162+
163+
/// The models `codex app-server` lists (`model/list`): a short-lived server
164+
/// of its own, asked once. The owner's apps and plugins stay off, as for a
165+
/// bot.
166+
pub fn models(program: &Path) -> io::Result<Vec<botloft_core::protocol::AgentModel>> {
167+
use std::io::{BufRead, BufReader, Write};
168+
use std::process::{Command, Stdio};
169+
use std::sync::mpsc;
170+
use std::time::Duration;
171+
172+
let mut child = Command::new(program)
173+
.args([
174+
"app-server",
175+
"-c",
176+
"features.apps=false",
177+
"-c",
178+
"features.plugins=false",
179+
])
180+
.stdin(Stdio::piped())
181+
.stdout(Stdio::piped())
182+
.stderr(Stdio::null())
183+
.spawn()?;
184+
let mut stdin = child
185+
.stdin
186+
.take()
187+
.ok_or_else(|| io::Error::other("no stdin"))?;
188+
let stdout = child
189+
.stdout
190+
.take()
191+
.ok_or_else(|| io::Error::other("no stdout"))?;
192+
let (lines, answers) = mpsc::channel::<String>();
193+
std::thread::spawn(move || {
194+
for line in BufReader::new(stdout).lines().map_while(Result::ok) {
195+
if lines.send(line).is_err() {
196+
break;
197+
}
198+
}
199+
});
200+
let send = |stdin: &mut std::process::ChildStdin, message: Value| {
201+
let mut line = message.to_string();
202+
line.push('\n');
203+
stdin.write_all(line.as_bytes())
204+
};
205+
let ask = |stdin: &mut std::process::ChildStdin, id: u64, method: &str, params: Value| {
206+
send(
207+
stdin,
208+
json!({ "id": id, "method": method, "params": params }),
209+
)?;
210+
loop {
211+
let line = answers
212+
.recv_timeout(Duration::from_secs(20))
213+
.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "codex did not answer"))?;
214+
let Ok(value) = serde_json::from_str::<Value>(&line) else {
215+
continue;
216+
};
217+
if value["id"] == id {
218+
return Ok(value);
219+
}
220+
}
221+
};
222+
let outcome = (|| -> io::Result<Value> {
223+
ask(
224+
&mut stdin,
225+
1,
226+
"initialize",
227+
json!({ "clientInfo": { "name": "botloft", "version": env!("CARGO_PKG_VERSION") } }),
228+
)?;
229+
send(&mut stdin, json!({ "method": "initialized", "params": {} }))?;
230+
ask(&mut stdin, 2, "model/list", json!({}))
231+
})();
232+
let _ = child.kill();
233+
let _ = child.wait();
234+
Ok(parse_models(&outcome?["result"]))
235+
}
236+
237+
fn parse_models(result: &Value) -> Vec<botloft_core::protocol::AgentModel> {
238+
result["data"]
239+
.as_array()
240+
.map(|models| {
241+
models
242+
.iter()
243+
.filter(|model| model["hidden"].as_bool() != Some(true))
244+
.filter_map(|model| {
245+
let id = model["model"].as_str().or_else(|| model["id"].as_str())?;
246+
Some(botloft_core::protocol::AgentModel {
247+
id: id.to_owned(),
248+
name: model["displayName"].as_str().unwrap_or(id).to_owned(),
249+
})
250+
})
251+
.collect()
252+
})
253+
.unwrap_or_default()
254+
}
255+
256+
#[cfg(test)]
257+
mod tests {
258+
use super::*;
259+
260+
#[test]
261+
fn the_list_skips_hidden_models_and_names_the_rest() {
262+
let models = parse_models(&json!({ "data": [
263+
{ "id": "a", "model": "gpt-a", "displayName": "GPT A", "hidden": false },
264+
{ "id": "b", "model": "gpt-b", "displayName": "GPT B", "hidden": true },
265+
{ "id": "c", "model": "gpt-c" },
266+
] }));
267+
let ids: Vec<&str> = models.iter().map(|m| m.id.as_str()).collect();
268+
assert_eq!(ids, ["gpt-a", "gpt-c"]);
269+
assert_eq!(models[0].name, "GPT A");
270+
assert_eq!(models[1].name, "gpt-c");
271+
}
272+
}

‎crates/botloftd/src/agent/codex_wire.rs‎

Lines changed: 54 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,8 @@ pub(super) struct State {
3939
next_id: u64,
4040
phase: Phase,
4141
pending: HashMap<u64, Pending>,
42-
/// Owner messages (text) waiting for the thread or for the running turn.
43-
queue: VecDeque<String>,
42+
/// The input of messages waiting for the thread or for the running turn.
43+
queue: VecDeque<Vec<Value>>,
4444
turn_active: bool,
4545
}
4646

@@ -125,12 +125,29 @@ impl Shared {
125125
pub(super) fn enqueue(&self, line: &[u8]) -> io::Result<()> {
126126
let event: Value = serde_json::from_slice(line)
127127
.map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?;
128-
let text = event["message"]["content"][0]["text"]
129-
.as_str()
130-
.unwrap_or_default()
131-
.to_owned();
128+
// Text as it is; an image as the data URL `turn/start` takes.
129+
let input: Vec<Value> = event["message"]["content"]
130+
.as_array()
131+
.map(|blocks| {
132+
blocks
133+
.iter()
134+
.filter_map(|block| match block["type"].as_str() {
135+
Some("text") => Some(json!({ "type": "text", "text": block["text"] })),
136+
Some("image") => Some(json!({
137+
"type": "image",
138+
"url": format!(
139+
"data:{};base64,{}",
140+
block["media_type"].as_str().unwrap_or("image/png"),
141+
block["data"].as_str().unwrap_or_default()
142+
),
143+
})),
144+
_ => None,
145+
})
146+
.collect()
147+
})
148+
.unwrap_or_default();
132149
let mut state = self.lock();
133-
state.queue.push_back(text);
150+
state.queue.push_back(input);
134151
self.pump(&mut state);
135152
Ok(())
136153
}
@@ -142,12 +159,12 @@ impl Shared {
142159
if state.turn_active {
143160
return;
144161
}
145-
let Some(text) = state.queue.pop_front() else {
162+
let Some(input) = state.queue.pop_front() else {
146163
return;
147164
};
148165
let mut params = json!({
149166
"threadId": thread,
150-
"input": [{ "type": "text", "text": text }],
167+
"input": input,
151168
});
152169
if let Some(effort) = &self.input.effort {
153170
params["effort"] = json!(effort);
@@ -383,4 +400,32 @@ mod tests {
383400
let last = written.lock().expect("lock").last().cloned().expect("line");
384401
assert_eq!(last["method"], "thread/start");
385402
}
403+
404+
#[test]
405+
fn an_image_goes_as_a_data_url_with_the_text() {
406+
let sink = Sink::default();
407+
let written = Arc::clone(&sink.0);
408+
let shared = Shared::begin(Box::new(sink), input(None));
409+
shared.answered(&json!({ "id": 1, "result": {} }));
410+
shared.answered(&json!({ "id": 2, "result": { "thread": { "id": "th" } } }));
411+
let line = json!({ "event": "user", "message": { "role": "user", "content": [
412+
{ "type": "text", "text": "look" },
413+
{ "type": "image", "media_type": "image/png", "data": "AAAA" },
414+
] } })
415+
.to_string();
416+
shared.enqueue(line.as_bytes()).expect("queued");
417+
let turn = written
418+
.lock()
419+
.expect("lock")
420+
.iter()
421+
.find(|line| line["method"] == "turn/start")
422+
.cloned()
423+
.expect("turn");
424+
assert_eq!(turn["params"]["input"][0]["text"], "look");
425+
assert_eq!(turn["params"]["input"][1]["type"], "image");
426+
assert_eq!(
427+
turn["params"]["input"][1]["url"],
428+
"data:image/png;base64,AAAA"
429+
);
430+
}
386431
}

‎crates/botloftd/src/agent/mod.rs‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ use serde_json::Value;
2121

2222
pub use agy::{AgyAgent, models as agy_models};
2323
pub use claude::ClaudeAgent;
24-
pub use codex::CodexAgent;
24+
pub use codex::{CodexAgent, models as codex_models};
2525

2626
use crate::runtime::ProcessControl;
2727
use crate::state::Daemon;
@@ -145,12 +145,21 @@ pub trait Agent: Send + Sync + 'static {
145145
}
146146
}
147147

148-
/// Botloft's own line for a turn, for agents whose stdin is a protocol of
149-
/// their own that a wrapper speaks: text only, as `agy` takes it.
150-
pub(crate) fn neutral_turn(turn: &Turn) -> Bytes {
148+
/// Botloft's own line for a turn, for an agent whose stdin is a protocol of
149+
/// its own that a wrapper speaks (Codex): the text, then each image as a block
150+
/// with its media type and base64 data.
151+
pub(crate) fn neutral_turn_with_images(turn: &Turn) -> Bytes {
152+
let mut content = vec![serde_json::json!({ "type": "text", "text": turn.text })];
153+
content.extend(turn.images.iter().map(|image| {
154+
serde_json::json!({
155+
"type": "image",
156+
"media_type": image.media_type,
157+
"data": image.data,
158+
})
159+
}));
151160
let event = serde_json::json!({
152161
"event": "user",
153-
"message": { "role": "user", "content": [{ "type": "text", "text": turn.text }] },
162+
"message": { "role": "user", "content": content },
154163
});
155164
let mut line = event.to_string().into_bytes();
156165
line.push(b'\n');

‎crates/botloftd/src/service/models.rs‎

Lines changed: 24 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -144,32 +144,41 @@ pub fn set_agent_model(daemon: &Daemon, params: BotsSetAgentModelParams) -> ApiR
144144
Ok(bot)
145145
}
146146

147-
/// The models the agent offers, as it lists them. Kept for a while: asking
148-
/// takes seconds.
147+
/// The models the agent offers, as it lists them. Kept for a while, for each
148+
/// agent: asking takes seconds.
149149
pub fn agent_models(daemon: &Daemon, params: AgentsModelsParams) -> ApiResult<Vec<AgentModel>> {
150150
use std::sync::Mutex;
151151
use std::time::{Duration, Instant};
152-
static CACHE: Mutex<Option<(Instant, Vec<AgentModel>)>> = Mutex::new(None);
153-
if params.agent != AgentKind::Agy {
154-
return Err(ApiError::validation(
155-
"only agy lists its models; Claude Code's are fixed",
156-
));
157-
}
152+
type Cached = Option<(Instant, Vec<AgentModel>)>;
153+
static CACHE: Mutex<[Cached; 2]> = Mutex::new([None, None]);
154+
let slot = match params.agent {
155+
AgentKind::Agy => 0,
156+
AgentKind::Codex => 1,
157+
AgentKind::Claude => {
158+
return Err(ApiError::validation(
159+
"Claude Code's models are fixed; only the other agents list theirs",
160+
));
161+
}
162+
};
158163
let mut cache = CACHE
159164
.lock()
160165
.unwrap_or_else(|poisoned| poisoned.into_inner());
161-
if let Some((at, models)) = cache.as_ref()
166+
if let Some((at, models)) = cache[slot].as_ref()
162167
&& at.elapsed() < Duration::from_secs(600)
163168
{
164169
return Ok(models.clone());
165170
}
166-
let agent = crate::agent::of(AgentKind::Agy)
167-
.ok_or_else(|| ApiError::validation("agy is not available"))?;
171+
let agent = crate::agent::of(params.agent)
172+
.ok_or_else(|| ApiError::validation("that agent is not available"))?;
168173
let program = agent
169-
.locate(daemon.supervisor.agy_path(), None)
174+
.locate(daemon.supervisor.program_setting(params.agent), None)
170175
.map_err(|err| ApiError::Conflict(err.to_string()))?;
171-
let models = crate::agent::agy_models(&program)
172-
.map_err(|err| ApiError::Conflict(format!("could not list the models: {err}")))?;
173-
*cache = Some((Instant::now(), models.clone()));
176+
let listed = match params.agent {
177+
AgentKind::Codex => crate::agent::codex_models(&program),
178+
_ => crate::agent::agy_models(&program),
179+
};
180+
let models =
181+
listed.map_err(|err| ApiError::Conflict(format!("could not list the models: {err}")))?;
182+
cache[slot] = Some((Instant::now(), models.clone()));
174183
Ok(models)
175184
}

0 commit comments

Comments
 (0)