diff --git a/VERSION b/VERSION index 3eefcb9..9084fa2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.0 +1.1.0 diff --git a/sdks/js/package.json b/sdks/js/package.json index 02cf550..e07b274 100644 --- a/sdks/js/package.json +++ b/sdks/js/package.json @@ -1,6 +1,6 @@ { "name": "@goreal-ai/plp-client", - "version": "1.2.0", + "version": "1.3.0", "description": "Official JavaScript/TypeScript client for PLP (Prompt Library Protocol)", "main": "dist/index.js", "module": "dist/index.mjs", diff --git a/sdks/js/src/index.ts b/sdks/js/src/index.ts index 8b053ac..3fee21a 100644 --- a/sdks/js/src/index.ts +++ b/sdks/js/src/index.ts @@ -105,6 +105,74 @@ export interface ResolvedContext { dataUrl: string | null; } +// ============================================================================= +// Deploy Types +// ============================================================================= + +export interface DeployRequest { + versionNo: number; + environment: string; +} + +export interface DeployResponse { + promptId: string; + versionNo: number; + environment: string; + deployedAt: string; +} + +// ============================================================================= +// Evaluation Types +// ============================================================================= + +export interface RunEvalRequest { + evalContent: string; + versionNo?: number; + datasetId?: string; +} + +export interface EvalSuiteResult { + suiteName: string; + status: 'pass' | 'fail' | 'error'; + tests: EvalTestResult[]; + summary: { + total: number; + passed: number; + failed: number; + durationMs: number; + }; +} + +export interface EvalTestResult { + name: string; + status: 'pass' | 'fail' | 'error'; + assertions: AssertionResult[]; + durationMs?: number; + error?: string; +} + +export interface AssertionResult { + operator: string; + status: 'pass' | 'fail' | 'error'; + expected?: string; + actual?: string; + message?: string; +} + +export interface EvalDatasetCase { + name: string; + input: Record; + expected?: string; + model?: string; + metadata?: Record; +} + +export interface EvalDataset { + id: string; + description?: string; + cases: EvalDatasetCase[]; +} + // ============================================================================= // Discovery Types // ============================================================================= @@ -117,6 +185,8 @@ export interface Discovery { list: boolean; search: boolean; contextStore?: boolean; + deploy?: boolean; + evaluation?: boolean; }; } @@ -405,6 +475,92 @@ export class PLPClient { return this.requestJson('POST', `/v1/prompts/${promptId}/publish`, body); } + // =========================================================================== + // Deploy Methods + // =========================================================================== + + /** + * Deploy a prompt version to a named environment + * @param promptId - The prompt identifier + * @param versionNo - The version number to deploy + * @param environment - Target environment (e.g., "staging", "production") + * @returns Deploy response with timestamp + */ + async deploy( + promptId: string, + versionNo: number, + environment: string + ): Promise { + const body: DeployRequest = { versionNo, environment }; + return this.requestJson( + 'POST', + `/v1/prompts/${promptId}/deploy`, + body + ); + } + + // =========================================================================== + // Evaluation Methods + // =========================================================================== + + /** + * Run an evaluation suite against a prompt + * @param promptId - The prompt identifier + * @param evalContent - The .eval file content (YAML format) + * @param options - Optional version and dataset references + * @returns Evaluation suite results + */ + async runEval( + promptId: string, + evalContent: string, + options?: { versionNo?: number; datasetId?: string } + ): Promise { + const body: RunEvalRequest = { + evalContent, + ...options, + }; + return this.requestJson( + 'POST', + `/v1/prompts/${promptId}/eval`, + body + ); + } + + /** + * Save or update an eval dataset + * @param promptId - The prompt identifier + * @param datasetId - The dataset identifier + * @param dataset - The dataset content + * @returns The saved dataset + */ + async saveDataset( + promptId: string, + datasetId: string, + dataset: EvalDataset + ): Promise { + return this.requestJson( + 'PUT', + `/v1/prompts/${promptId}/eval/datasets/${datasetId}`, + dataset + ); + } + + /** + * Retrieve an eval dataset + * @param promptId - The prompt identifier + * @param datasetId - The dataset identifier + * @returns The dataset + */ + async getDataset( + promptId: string, + datasetId: string + ): Promise { + return this.request( + 'GET', + `/v1/prompts/${promptId}/eval/datasets/${datasetId}` + ); + } + // =========================================================================== // Context Store Methods // =========================================================================== diff --git a/sdks/python/pyproject.toml b/sdks/python/pyproject.toml index db431d4..99eab68 100644 --- a/sdks/python/pyproject.toml +++ b/sdks/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "plp-client" -version = "1.0.0" +version = "1.1.0" description = "Official Python client for PLP (Prompt Library Protocol)" readme = "README.md" requires-python = ">=3.9" diff --git a/sdks/python/src/plp_client/__init__.py b/sdks/python/src/plp_client/__init__.py index e70c915..69fc66b 100644 --- a/sdks/python/src/plp_client/__init__.py +++ b/sdks/python/src/plp_client/__init__.py @@ -26,7 +26,7 @@ get_text_content, ) -__version__ = "1.0.0" +__version__ = "1.1.0" __all__ = [ "PLPClient", "PLPError", diff --git a/sdks/python/src/plp_client/client.py b/sdks/python/src/plp_client/client.py index 22aab10..985673f 100644 --- a/sdks/python/src/plp_client/client.py +++ b/sdks/python/src/plp_client/client.py @@ -319,6 +319,179 @@ def from_dict(cls, data: Dict[str, Any]) -> "ResolvedContext": ) +# ============================================================================= +# Deploy Types +# ============================================================================= + + +@dataclass +class DeployResponse: + """Response from deploying a prompt version.""" + + prompt_id: str + version_no: int + environment: str + deployed_at: str + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "DeployResponse": + """Create from dictionary.""" + return cls( + prompt_id=data["promptId"], + version_no=data["versionNo"], + environment=data["environment"], + deployed_at=data["deployedAt"], + ) + + +# ============================================================================= +# Evaluation Types +# ============================================================================= + + +@dataclass +class AssertionResult: + """Result of a single assertion.""" + + operator: str + status: str # "pass" | "fail" | "error" + expected: Optional[str] = None + actual: Optional[str] = None + message: Optional[str] = None + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "AssertionResult": + """Create from dictionary.""" + return cls( + operator=data["operator"], + status=data["status"], + expected=data.get("expected"), + actual=data.get("actual"), + message=data.get("message"), + ) + + +@dataclass +class EvalTestResult: + """Result of a single eval test.""" + + name: str + status: str # "pass" | "fail" | "error" + assertions: List[AssertionResult] + duration_ms: Optional[float] = None + error: Optional[str] = None + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "EvalTestResult": + """Create from dictionary.""" + return cls( + name=data["name"], + status=data["status"], + assertions=[AssertionResult.from_dict(a) for a in data["assertions"]], + duration_ms=data.get("durationMs"), + error=data.get("error"), + ) + + +@dataclass +class EvalSummary: + """Summary of eval suite results.""" + + total: int + passed: int + failed: int + duration_ms: float + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "EvalSummary": + """Create from dictionary.""" + return cls( + total=data["total"], + passed=data["passed"], + failed=data["failed"], + duration_ms=data["durationMs"], + ) + + +@dataclass +class EvalSuiteResult: + """Result of running an eval suite.""" + + suite_name: str + status: str # "pass" | "fail" | "error" + tests: List[EvalTestResult] + summary: EvalSummary + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "EvalSuiteResult": + """Create from dictionary.""" + return cls( + suite_name=data["suiteName"], + status=data["status"], + tests=[EvalTestResult.from_dict(t) for t in data["tests"]], + summary=EvalSummary.from_dict(data["summary"]), + ) + + +@dataclass +class EvalDatasetCase: + """A single test case in a dataset.""" + + name: str + input: Dict[str, Any] + expected: Optional[str] = None + model: Optional[str] = None + metadata: Optional[Dict[str, Any]] = None + + def to_dict(self) -> Dict[str, Any]: + result: Dict[str, Any] = {"name": self.name, "input": self.input} + if self.expected is not None: + result["expected"] = self.expected + if self.model is not None: + result["model"] = self.model + if self.metadata is not None: + result["metadata"] = self.metadata + return result + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "EvalDatasetCase": + """Create from dictionary.""" + return cls( + name=data["name"], + input=data["input"], + expected=data.get("expected"), + model=data.get("model"), + metadata=data.get("metadata"), + ) + + +@dataclass +class EvalDataset: + """An evaluation dataset with test cases.""" + + id: str + cases: List[EvalDatasetCase] + description: Optional[str] = None + + def to_dict(self) -> Dict[str, Any]: + result: Dict[str, Any] = { + "id": self.id, + "cases": [c.to_dict() for c in self.cases], + } + if self.description is not None: + result["description"] = self.description + return result + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "EvalDataset": + """Create from dictionary.""" + return cls( + id=data["id"], + cases=[EvalDatasetCase.from_dict(c) for c in data["cases"]], + description=data.get("description"), + ) + + class PLPError(Exception): """Exception raised for PLP-related errors.""" @@ -597,6 +770,105 @@ def resolve_prompt_context( ) return {name: ResolvedContext.from_dict(value) for name, value in data.items()} + # ========================================================================= + # Deploy Methods + # ========================================================================= + + def deploy( + self, prompt_id: str, version_no: int, environment: str + ) -> DeployResponse: + """ + Deploy a prompt version to a named environment. + + Args: + prompt_id: The prompt identifier + version_no: The version number to deploy + environment: Target environment (e.g., "staging", "production") + + Returns: + DeployResponse with deployment details + """ + data = self._request( + "POST", + f"/v1/prompts/{prompt_id}/deploy", + json={"versionNo": version_no, "environment": environment}, + ) + return DeployResponse.from_dict(data) + + # ========================================================================= + # Evaluation Methods + # ========================================================================= + + def run_eval( + self, + prompt_id: str, + eval_content: str, + version_no: Optional[int] = None, + dataset_id: Optional[str] = None, + ) -> EvalSuiteResult: + """ + Run an evaluation suite against a prompt. + + Args: + prompt_id: The prompt identifier + eval_content: The .eval file content (YAML format) + version_no: Optional prompt version (default: latest) + dataset_id: Optional reference to a saved dataset + + Returns: + EvalSuiteResult with test results and summary + """ + body: Dict[str, Any] = {"evalContent": eval_content} + if version_no is not None: + body["versionNo"] = version_no + if dataset_id is not None: + body["datasetId"] = dataset_id + + data = self._request( + "POST", + f"/v1/prompts/{prompt_id}/eval", + json=body, + ) + return EvalSuiteResult.from_dict(data) + + def save_dataset( + self, prompt_id: str, dataset_id: str, dataset: EvalDataset + ) -> EvalDataset: + """ + Save or update an eval dataset. + + Args: + prompt_id: The prompt identifier + dataset_id: The dataset identifier + dataset: The dataset to save + + Returns: + The saved dataset + """ + data = self._request( + "PUT", + f"/v1/prompts/{prompt_id}/eval/datasets/{dataset_id}", + json=dataset.to_dict(), + ) + return EvalDataset.from_dict(data) + + def get_dataset(self, prompt_id: str, dataset_id: str) -> EvalDataset: + """ + Retrieve an eval dataset. + + Args: + prompt_id: The prompt identifier + dataset_id: The dataset identifier + + Returns: + The dataset + """ + data = self._request( + "GET", + f"/v1/prompts/{prompt_id}/eval/datasets/{dataset_id}", + ) + return EvalDataset.from_dict(data) + def __enter__(self) -> "PLPClient": """Context manager entry.""" return self diff --git a/spec/SPEC.md b/spec/SPEC.md index 768aa08..780a7ba 100644 --- a/spec/SPEC.md +++ b/spec/SPEC.md @@ -314,6 +314,140 @@ curl -X DELETE https://prompts.example.com/v1/prompts/deprecated/old-prompt --- +## 3.5. Deploy Prompt (POST) + +Deploys a specific version of a prompt to a named environment. + +### Endpoint + +```http +POST /v1/prompts/{prompt_id}/deploy +``` + +### Request Body + +```json +{ + "versionNo": 3, + "environment": "production" +} +``` + +### Fields + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `versionNo` | Integer | REQUIRED | The version number to deploy | +| `environment` | String | REQUIRED | Target environment name (e.g., `staging`, `production`) | + +### Response (200 OK) + +```json +{ + "promptId": "marketing/welcome-email", + "versionNo": 3, + "environment": "production", + "deployedAt": "2025-06-15T14:30:00Z" +} +``` + +### Discovery + +Servers that support deployment MUST set `capabilities.deploy: true` in the discovery endpoint. + +--- + +## 3.6. Evaluation Endpoints + +PLP provides endpoints for running prompt evaluations and managing test datasets. These enable automated testing of prompts against expected behaviors. + +### Discovery + +Servers that support evaluation MUST set `capabilities.evaluation: true` in the discovery endpoint. + +### Run Evaluation + +```http +POST /v1/prompts/{prompt_id}/eval +``` + +Runs an evaluation suite against a prompt. The eval content (YAML format) is provided in the request body. + +#### Request Body + +```json +{ + "evalContent": "suite: Smoke Tests\ntests:\n - name: basic test\n ...", + "versionNo": 3, + "datasetId": "family-scenarios" +} +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `evalContent` | String | REQUIRED | The `.eval` file content (YAML) | +| `versionNo` | Integer | OPTIONAL | Prompt version (default: latest) | +| `datasetId` | String | OPTIONAL | Reference to a saved dataset | + +#### Response (200 OK) + +```json +{ + "suiteName": "Smoke Tests", + "status": "fail", + "tests": [ + { + "name": "renders family-friendly rules", + "status": "pass", + "assertions": [ + { "operator": "contains", "status": "pass" } + ] + } + ], + "summary": { + "total": 5, + "passed": 4, + "failed": 1, + "durationMs": 3200 + } +} +``` + +### Save/Update Dataset + +```http +PUT /v1/prompts/{prompt_id}/eval/datasets/{dataset_id} +``` + +Saves a dataset of test cases with optional golden responses. + +### Retrieve Dataset + +```http +GET /v1/prompts/{prompt_id}/eval/datasets/{dataset_id} +``` + +Retrieves a saved evaluation dataset. + +#### Dataset Format + +```json +{ + "id": "family-scenarios", + "description": "Family-friendly scenarios", + "cases": [ + { + "name": "family-comedy", + "input": { "companions": "Family", "genre": "Comedy" }, + "expected": "I recommend Finding Nemo...", + "model": "gpt-4o" + } + ] +} +``` + +--- + ## 4. HTTP Headers ### Required Request Headers diff --git a/spec/openapi.yaml b/spec/openapi.yaml index a657bfe..b427420 100644 --- a/spec/openapi.yaml +++ b/spec/openapi.yaml @@ -26,6 +26,10 @@ tags: description: Server discovery and capabilities - name: prompts description: Prompt CRUD operations + - name: deploy + description: Prompt deployment to named environments + - name: evaluation + description: Prompt evaluation and testing - name: context-store description: Context Store for user's private assets (images, text files) - name: prompt-context @@ -422,6 +426,217 @@ paths: schema: $ref: '#/components/schemas/Error' + # Deploy Endpoint + /prompts/{prompt_id}/deploy: + post: + tags: + - deploy + summary: Deploy a prompt version to an environment + description: | + Deploys a specific version of a prompt to a named environment (e.g., staging, production). + This pins that version as the active version for the given environment. + operationId: deployPrompt + parameters: + - name: prompt_id + in: path + required: true + description: The prompt identifier + schema: + type: string + example: marketing/welcome-email + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/DeployRequest' + example: + versionNo: 3 + environment: production + responses: + '200': + description: Successfully deployed + content: + application/json: + schema: + $ref: '#/components/schemas/DeployResponse' + example: + promptId: "marketing/welcome-email" + versionNo: 3 + environment: "production" + deployedAt: "2025-06-15T14:30:00Z" + '400': + description: Invalid request (unknown environment or version) + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '401': + description: Unauthorized + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '404': + description: Prompt or version not found + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + + # Evaluation Endpoints + /prompts/{prompt_id}/eval: + post: + tags: + - evaluation + summary: Run an evaluation against a prompt + description: | + Runs an evaluation suite against a prompt. The eval content is provided as + YAML (the .eval file format) and optionally references a saved dataset. + Returns structured results with pass/fail status for each test and assertion. + operationId: runEval + parameters: + - name: prompt_id + in: path + required: true + description: The prompt identifier + schema: + type: string + example: marketing/welcome-email + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/RunEvalRequest' + responses: + '200': + description: Evaluation completed (may contain failures) + content: + application/json: + schema: + $ref: '#/components/schemas/EvalSuiteResult' + '400': + description: Invalid eval content or configuration + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '401': + description: Unauthorized + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '404': + description: Prompt not found + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + + /prompts/{prompt_id}/eval/datasets/{dataset_id}: + put: + tags: + - evaluation + summary: Save or update an eval dataset + description: | + Saves a dataset containing test cases with golden responses for evaluation. + Creates the dataset if it doesn't exist, or replaces it if it does. + operationId: saveEvalDataset + parameters: + - name: prompt_id + in: path + required: true + description: The prompt identifier + schema: + type: string + - name: dataset_id + in: path + required: true + description: The dataset identifier + schema: + type: string + example: family-scenarios + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/EvalDataset' + responses: + '200': + description: Dataset updated + content: + application/json: + schema: + $ref: '#/components/schemas/EvalDataset' + '201': + description: Dataset created + content: + application/json: + schema: + $ref: '#/components/schemas/EvalDataset' + '400': + description: Invalid dataset format + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '401': + description: Unauthorized + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '404': + description: Prompt not found + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + + get: + tags: + - evaluation + summary: Retrieve an eval dataset + description: | + Retrieves a saved evaluation dataset by ID. + operationId: getEvalDataset + parameters: + - name: prompt_id + in: path + required: true + description: The prompt identifier + schema: + type: string + - name: dataset_id + in: path + required: true + description: The dataset identifier + schema: + type: string + example: family-scenarios + responses: + '200': + description: Dataset retrieved + content: + application/json: + schema: + $ref: '#/components/schemas/EvalDataset' + '401': + description: Unauthorized + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '404': + description: Prompt or dataset not found + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + # Context Store Endpoints /context-store: get: @@ -958,6 +1173,14 @@ components: type: boolean description: Whether Context Store is supported example: true + deploy: + type: boolean + description: Whether prompt deployment to environments is supported + example: true + evaluation: + type: boolean + description: Whether prompt evaluation/testing is supported + example: true additionalProperties: true # Context Store Schemas @@ -1216,3 +1439,202 @@ components: type: integer description: The published version number example: 3 + + # Deploy Schemas + DeployRequest: + type: object + required: + - versionNo + - environment + properties: + versionNo: + type: integer + description: The version number to deploy + minimum: 1 + example: 3 + environment: + type: string + description: Target environment name (e.g., staging, production) + example: production + + DeployResponse: + type: object + required: + - promptId + - versionNo + - environment + - deployedAt + properties: + promptId: + type: string + description: The prompt identifier + example: marketing/welcome-email + versionNo: + type: integer + description: The deployed version number + example: 3 + environment: + type: string + description: The target environment + example: production + deployedAt: + type: string + format: date-time + description: When the deployment occurred + example: "2025-06-15T14:30:00Z" + + # Evaluation Schemas + RunEvalRequest: + type: object + required: + - evalContent + properties: + evalContent: + type: string + description: The .eval file content (YAML format) + versionNo: + type: integer + description: Prompt version to evaluate (default latest) + minimum: 1 + datasetId: + type: string + description: Reference to a saved dataset for test inputs + + EvalSuiteResult: + type: object + required: + - suiteName + - status + - tests + - summary + properties: + suiteName: + type: string + description: Name of the eval suite + example: Smoke Tests + status: + type: string + enum: [pass, fail, error] + description: Overall suite status + tests: + type: array + items: + $ref: '#/components/schemas/EvalTestResult' + summary: + type: object + required: + - total + - passed + - failed + - durationMs + properties: + total: + type: integer + description: Total number of tests + example: 5 + passed: + type: integer + description: Number of passed tests + example: 4 + failed: + type: integer + description: Number of failed tests + example: 1 + durationMs: + type: number + description: Total duration in milliseconds + example: 3200 + + EvalTestResult: + type: object + required: + - name + - status + - assertions + properties: + name: + type: string + description: Test name + example: renders family-friendly rules + status: + type: string + enum: [pass, fail, error] + description: Test status + assertions: + type: array + items: + $ref: '#/components/schemas/AssertionResult' + durationMs: + type: number + description: Test duration in milliseconds + error: + type: string + description: Error message if status is error + + AssertionResult: + type: object + required: + - operator + - status + properties: + operator: + type: string + description: The assertion operator used + example: contains + status: + type: string + enum: [pass, fail, error] + description: Assertion status + expected: + type: string + description: Expected value + actual: + type: string + description: Actual value + message: + type: string + description: Human-readable result message + + EvalDataset: + type: object + required: + - id + - cases + properties: + id: + type: string + description: Unique dataset identifier + example: family-scenarios + description: + type: string + description: Human-readable description + cases: + type: array + items: + $ref: '#/components/schemas/EvalDatasetCase' + description: Test cases with inputs and optional expected outputs + + EvalDatasetCase: + type: object + required: + - name + - input + properties: + name: + type: string + description: Test case name + example: family-comedy + input: + type: object + additionalProperties: true + description: Prompt variables as key-value pairs + expected: + type: string + description: Golden/reference response + model: + type: string + description: Model used to generate the golden response + metadata: + type: object + additionalProperties: true + description: Additional metadata (tokens, latency, etc.)