Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 31 additions & 18 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -135,24 +135,26 @@ Remove-Item -Recurse -Force (Join-Path $env:APPDATA "bt") -ErrorAction SilentlyC

## Commands

| Command | Description |
| ------------- | ------------------------------------------------------------------ |
| `bt init` | Initialize `.bt/` config directory and link to a project |
| `bt login` | Log in to Braintrust or refresh an OAuth login |
| `bt logout` | Remove a saved Braintrust login |
| `bt profiles` | List, delete, and rename saved login profiles |
| `bt switch` | Switch org and project context |
| `bt status` | Show current org and project context |
| `bt datasets` | Manage datasets and dataset pipelines |
| `bt eval` | Run eval files (Unix only) |
| `bt sql` | Run SQL queries against Braintrust |
| `bt view` | View logs, traces, and spans |
| `bt projects` | Manage projects (list, create, view, delete) |
| `bt datasets` | Manage remote datasets (list, create, update, view, delete) |
| `bt prompts` | Manage prompts (list, view, delete) |
| `bt scorers` | Manage scorers (list, create, view, invoke, delete) |
| `bt sync` | Synchronize project logs between Braintrust and local NDJSON files |
| `bt update` | Update bt in-place |
| Command | Description |
| -------------- | ------------------------------------------------------------------ |
| `bt init` | Initialize `.bt/` config directory and link to a project |
| `bt login` | Log in to Braintrust or refresh an OAuth login |
| `bt logout` | Remove a saved Braintrust login |
| `bt profiles` | List, delete, and rename saved login profiles |
| `bt switch` | Switch org and project context |
| `bt status` | Show current org and project context |
| `bt datasets` | Manage datasets and dataset pipelines |
| `bt eval` | Run eval files (Unix only) |
| `bt sql` | Run SQL queries against Braintrust |
| `bt view` | View logs, traces, and spans |
| `bt projects` | Manage projects (list, create, view, delete) |
| `bt datasets` | Manage remote datasets (list, create, update, view, delete) |
| `bt prompts` | Manage prompts (list, view, delete) |
| `bt functions` | Manage functions (list, view, invoke, update, push, pull, delete) |
| `bt tools` | Manage tools (list, view, invoke, update, delete) |
| `bt scorers` | Manage scorers (list, create, view, invoke, update, delete) |
| `bt sync` | Synchronize project logs between Braintrust and local NDJSON files |
| `bt update` | Update bt in-place |

## `bt scorers`

Expand All @@ -175,6 +177,17 @@ Use `--if-exists error|ignore|replace` to control slug conflicts. Text and struc

Before writing a scorer, `bt` sends the complete candidate definition to Braintrust for validation. The backend applies the same model-parameter and replacement checks as the write and returns structured issues with normalization suggestions when available.

Update only the fields you specify, or use `--patch` for fields without dedicated flags:

```bash
bt scorers update helpfulness --messages @messages.json
bt scorers update helpfulness --new-slug answer-helpfulness
bt functions update my-function --name "Updated function" --description "Updated"
bt tools update my-tool --prompt @prompt.txt --new-slug lookup-order
```

The API replaces `prompt_data` rather than merging it, so `bt` reads the current definition and sends a materialized replacement with your changes. A concurrent edit can therefore be overwritten.

For TypeScript and Python code scorers, use the Braintrust SDK and `bt functions push`.

## `bt eval`
Expand Down
14 changes: 14 additions & 0 deletions src/functions/api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,20 @@ pub async fn delete_function(client: &ApiClient, function_id: &str) -> Result<()
client.delete(&path).await
}

/// Partially update a function (scorer/tool/prompt/...) by id.
///
/// Top-level fields are patched, but object-valued fields such as `prompt_data`
/// are replaced wholesale. Callers updating `prompt_data` must materialize the
/// complete value before sending the request.
pub async fn patch_function(
client: &ApiClient,
function_id: &str,
body: &serde_json::Value,
) -> Result<serde_json::Value> {
let path = format!("/v1/function/{}", encode(function_id));
client.patch(&path, body).await
}

pub async fn list_functions_page(
client: &ApiClient,
query: &FunctionListQuery,
Expand Down
109 changes: 23 additions & 86 deletions src/functions/create.rs
Original file line number Diff line number Diff line change
@@ -1,20 +1,17 @@
use anyhow::{bail, Context, Result};
use clap::{builder::BoolishValueParser, ArgGroup, Args};
use dialoguer::Input;
use serde_json::{json, Map, Value};
use serde_json::{json, Value};

use crate::{
error::user_error,
ui::{is_interactive, print_command_status, with_spinner, CommandStatus},
utils::{merge_json_objects, read_text_source, read_yaml_object_source},
};

use super::{
api,
prompt_config::{
parse_choice_scores_source, parse_classifications_source, validate_unit_interval,
PromptConfigArgs,
},
prompt_config::PromptConfigArgs,
scorer_config::{build_scorer_config, ScorerConfig},
IfExistsMode, ResolvedContext,
};

Expand Down Expand Up @@ -275,101 +272,41 @@ fn build_scorer_definition(
name: &str,
slug: &str,
) -> Result<Value> {
let prompt = resolve_prompt_block(args)?;
let (function_type, parser) = resolve_output_parser(args)?;

let mut prompt_data = json!({
"prompt": prompt,
"parser": parser,
})
.as_object()
.expect("prompt data is an object")
.clone();
let prompt_config = args
.prompt_config
.build_prompt_data_patch(Some(&args.model))?;
merge_json_objects(&mut prompt_data, &prompt_config);
let config = build_scorer_config(
&ScorerConfig {
messages: Some(&args.messages),
model: Some(&args.model),
prompt_config: &args.prompt_config,
choice_scores: args.choice_scores.as_deref(),
classifications: args.classifications.as_deref(),
use_cot: Some(args.use_cot),
allow_no_match: args.classifications.as_ref().map(|_| args.allow_no_match),
pass_threshold: args.pass_threshold,
metadata: args.metadata.as_deref(),
metadata_label: "scorer metadata",
},
true,
)?;

let mut definition = json!({
"project_id": project_id,
"name": name,
"slug": slug,
"function_data": {
"type": "prompt",
},
"prompt_data": prompt_data,
"function_data": { "type": "prompt" },
"if_exists": args.if_exists.as_str(),
"function_type": function_type,
});
definition
.as_object_mut()
.expect("scorer definition is an object")
.extend(config);

if let Some(description) = args.description.as_deref() {
definition["description"] = Value::String(description.to_string());
}

let metadata = resolve_metadata(args)?;
if !metadata.is_empty() {
definition["metadata"] = Value::Object(metadata);
}

Ok(definition)
}

fn resolve_output_parser(args: &CreateArgs) -> Result<(&'static str, Value)> {
match (
args.choice_scores.as_deref(),
args.classifications.as_deref(),
) {
(Some(source), None) => Ok((
"scorer",
json!({
"type": "llm_classifier",
"use_cot": args.use_cot,
"choice_scores": parse_choice_scores_source(source)?,
}),
)),
(None, Some(source)) => Ok((
"classifier",
json!({
"type": "llm_classifier",
"use_cot": args.use_cot,
"choice": parse_classifications_source(source)?,
"allow_no_match": args.allow_no_match,
}),
)),
(Some(_), Some(_)) => bail!(
"use either --choice-scores for score output or --classifications for classification output, not both"
),
(None, None) => bail!(
"output choices required. Pass --choice-scores <SOURCE> or --classifications <SOURCE>"
),
}
}

fn resolve_metadata(args: &CreateArgs) -> Result<Map<String, Value>> {
let mut metadata = match args.metadata.as_deref() {
Some(source) => read_yaml_object_source(source, "scorer metadata")?,
None => Map::new(),
};
if let Some(pass_threshold) = args.pass_threshold {
validate_unit_interval(pass_threshold, "--pass-threshold")?;
metadata.insert("__pass_threshold".to_string(), json!(pass_threshold));
}
Ok(metadata)
}

fn resolve_prompt_block(args: &CreateArgs) -> Result<Value> {
let raw = read_text_source(&args.messages, "messages")?;
parse_messages(&raw)
}

fn parse_messages(raw: &str) -> Result<Value> {
let messages: Value = serde_json::from_str(raw).context("invalid JSON in scorer messages")?;
match messages {
Value::Array(_) => Ok(json!({ "type": "chat", "messages": messages })),
_ => bail!("scorer messages must be a JSON array"),
}
}

#[cfg(test)]
mod tests {
use clap::Parser;
Expand Down
Loading
Loading