diff --git a/README.md b/README.md index 66b3e5b..18c3334 100644 --- a/README.md +++ b/README.md @@ -209,9 +209,17 @@ rli axon events # List events for an axon ```bash rli scenario info # Display scenario definition details +rli scenario create # Create a new custom scenario rli scenario list # List scenario runs ``` +### Benchmark-run Commands (alias: `bmr`) + +```bash +rli benchmark-run cancel # Cancel a running benchmark run +rli benchmark-run complete # Complete a benchmark run (finalize an... +``` + ### Benchmark-job Commands (alias: `bmj`) ```bash diff --git a/src/commands/benchmark-job/list.ts b/src/commands/benchmark-job/list.ts index 764e2ce..de5d677 100644 --- a/src/commands/benchmark-job/list.ts +++ b/src/commands/benchmark-job/list.ts @@ -6,9 +6,10 @@ import { listBenchmarkJobs, type BenchmarkJob, } from "../../services/benchmarkJobService.js"; -import { output, outputError } from "../../utils/output.js"; +import { output, outputError, parseLimit } from "../../utils/output.js"; interface ListOptions { + limit?: string; days?: string; all?: boolean; status?: string; @@ -30,18 +31,20 @@ const PAGE_SIZE = 100; async function fetchJobs( cutoffMs: number | null, statusFilter: Set | null, + maxResults: number, ): Promise { const allJobs: BenchmarkJob[] = []; let cursor: string | undefined; - while (true) { + while (allJobs.length < maxResults) { + const remaining = maxResults - allJobs.length; const result = await listBenchmarkJobs({ - limit: PAGE_SIZE, + limit: Math.min(PAGE_SIZE, remaining), startingAfter: cursor, }); for (const job of result.jobs) { - // Stop pagination if we've passed the time cutoff (API returns newest-first) + if (allJobs.length >= maxResults) break; if (cutoffMs !== null && job.create_time_ms < cutoffMs) { return applyStatusFilter(allJobs, statusFilter); } @@ -94,8 +97,10 @@ export async function listBenchmarkJobsCommand( cutoffMs = Date.now() - days * 86_400_000; } + const maxResults = parseLimit(options.limit); + // Fetch and filter - const jobs = await fetchJobs(cutoffMs, statusFilter); + const jobs = await fetchJobs(cutoffMs, statusFilter, maxResults); // Sort ascending by create_time_ms (oldest first, most recent at bottom) jobs.sort((a, b) => a.create_time_ms - b.create_time_ms); diff --git a/src/commands/benchmark-run/cancel.ts b/src/commands/benchmark-run/cancel.ts new file mode 100644 index 0000000..1e3e6c3 --- /dev/null +++ b/src/commands/benchmark-run/cancel.ts @@ -0,0 +1,14 @@ +import { cancelBenchmarkRun } from "../../services/benchmarkService.js"; +import { output, outputError } from "../../utils/output.js"; + +export async function cancelBenchmarkRunCommand( + id: string, + options: { output?: string }, +) { + try { + const result = await cancelBenchmarkRun(id); + output(result, { format: options.output, defaultFormat: "json" }); + } catch (error) { + outputError("Failed to cancel benchmark run", error); + } +} diff --git a/src/commands/benchmark-run/complete.ts b/src/commands/benchmark-run/complete.ts new file mode 100644 index 0000000..6c50733 --- /dev/null +++ b/src/commands/benchmark-run/complete.ts @@ -0,0 +1,14 @@ +import { completeBenchmarkRun } from "../../services/benchmarkService.js"; +import { output, outputError } from "../../utils/output.js"; + +export async function completeBenchmarkRunCommand( + id: string, + options: { output?: string }, +) { + try { + const result = await completeBenchmarkRun(id); + output(result, { format: options.output, defaultFormat: "json" }); + } catch (error) { + outputError("Failed to complete benchmark run", error); + } +} diff --git a/src/commands/blueprint/list.tsx b/src/commands/blueprint/list.tsx index 065c04e..7540ebc 100644 --- a/src/commands/blueprint/list.tsx +++ b/src/commands/blueprint/list.tsx @@ -659,6 +659,7 @@ const ListBlueprintsUI = ({ executeOperation(selectedBlueprintItem, "view_logs"); } }, + c: () => navigate("blueprint-create"), o: handleOpenInBrowser, "/": () => search.enterSearchMode(), escape: handleListEscape, @@ -1008,6 +1009,7 @@ const ListBlueprintsUI = ({ condition: hasMore || hasPrev, }, { key: "Enter", label: "Details" }, + { key: "c", label: "Create" }, { key: "a", label: "Actions" }, { key: "Tab", label: "Switch tab" }, { key: "o", label: "Browser" }, diff --git a/src/commands/scenario/create.ts b/src/commands/scenario/create.ts new file mode 100644 index 0000000..a2394f7 --- /dev/null +++ b/src/commands/scenario/create.ts @@ -0,0 +1,117 @@ +import { readFile } from "fs/promises"; +import { createScenario } from "../../services/scenarioService.js"; +import { output, outputError } from "../../utils/output.js"; +import { parseMetadata } from "../../utils/metadata.js"; +import type { ScenarioCreateParams } from "@runloop/api-client/resources/scenarios/scenarios"; + +interface CreateScenarioOptions { + name: string; + problemStatement: string; + scoringCommand?: string; + scoringFile?: string; + blueprint?: string; + snapshot?: string; + workingDirectory?: string; + referenceOutput?: string; + referenceOutputFile?: string; + metadata?: string[]; + requiredEnvVars?: string[]; + requiredSecrets?: string[]; + scorerTimeout?: string; + validationType?: string; + output?: string; +} + +export async function createScenarioCommand(options: CreateScenarioOptions) { + try { + if (!options.scoringCommand && !options.scoringFile) { + return outputError( + "At least one of --scoring-command or --scoring-file is required", + ); + } + + let scoringContract: ScenarioCreateParams["scoring_contract"]; + + if (options.scoringFile) { + const contents = await readFile(options.scoringFile, "utf-8"); + scoringContract = JSON.parse(contents); + } else { + scoringContract = { + scoring_function_parameters: [ + { + name: "default", + weight: 1.0, + scorer: { + type: "command_scorer" as const, + command: options.scoringCommand!, + }, + }, + ], + }; + } + + const params: ScenarioCreateParams = { + name: options.name, + input_context: { + problem_statement: options.problemStatement, + }, + scoring_contract: scoringContract, + }; + + if (options.blueprint || options.snapshot || options.workingDirectory) { + params.environment_parameters = {}; + if (options.blueprint) { + params.environment_parameters.blueprint_id = options.blueprint; + } + if (options.snapshot) { + params.environment_parameters.snapshot_id = options.snapshot; + } + if (options.workingDirectory) { + params.environment_parameters.working_directory = + options.workingDirectory; + } + } + + if (options.referenceOutputFile) { + params.reference_output = await readFile( + options.referenceOutputFile, + "utf-8", + ); + } else if (options.referenceOutput) { + params.reference_output = options.referenceOutput; + } + + if (options.metadata) { + params.metadata = parseMetadata(options.metadata); + } + + if (options.requiredEnvVars) { + params.required_environment_variables = options.requiredEnvVars; + } + + if (options.requiredSecrets) { + params.required_secret_names = options.requiredSecrets; + } + + if (options.scorerTimeout) { + const timeout = parseInt(options.scorerTimeout, 10); + if (isNaN(timeout)) { + return outputError("--scorer-timeout must be a number"); + } + params.scorer_timeout_sec = timeout; + } + + if (options.validationType) { + params.validation_type = options.validationType as + | "UNSPECIFIED" + | "FORWARD" + | "REVERSE" + | "EVALUATION"; + } + + const scenario = await createScenario(params); + output(scenario, { format: options.output, defaultFormat: "json" }); + } catch (error) { + outputError("Failed to create scenario", error); + } +} diff --git a/src/components/BlueprintCreatePage.tsx b/src/components/BlueprintCreatePage.tsx new file mode 100644 index 0000000..76c3e45 --- /dev/null +++ b/src/components/BlueprintCreatePage.tsx @@ -0,0 +1,758 @@ +import React from "react"; +import { Box, Text, useInput } from "ink"; +import TextInput from "ink-text-input"; +import figures from "figures"; +import { SpinnerComponent } from "./Spinner.js"; +import { ErrorMessage } from "./ErrorMessage.js"; +import { Breadcrumb } from "./Breadcrumb.js"; +import { NavigationTips } from "./NavigationTips.js"; +import { MetadataDisplay } from "./MetadataDisplay.js"; +import { + FormTextInput, + FormSelect, + FormActionButton, + FormListManager, + useFormSelectNavigation, +} from "./form/index.js"; +import { colors } from "../utils/theme.js"; +import { useExitOnCtrlC } from "../hooks/useExitOnCtrlC.js"; +import { createBlueprint, getBlueprint } from "../services/blueprintService.js"; + +interface BlueprintCreatePageProps { + onBack: () => void; + onCreate: (blueprintId: string) => void; + baseBlueprintId?: string; +} + +type ScreenState = "form" | "loading-base" | "creating" | "error"; + +type SourceType = "dockerfile" | "base_blueprint"; + +const SOURCE_TYPE_OPTIONS = ["dockerfile", "base_blueprint"] as const; +const ARCHITECTURE_OPTIONS = ["x86_64", "arm64"] as const; +const RESOURCE_OPTIONS = ["SMALL", "MEDIUM", "LARGE", "X_LARGE"] as const; + +interface FormData { + name: string; + sourceType: SourceType; + dockerfile: string; + baseBlueprintId: string; + systemSetupCommands: string[]; + architecture: string; + resources: string; + metadata: Record; +} + +type FieldKey = + | "submit" + | "name" + | "sourceType" + | "dockerfile" + | "baseBlueprintId" + | "setupCommands" + | "architecture" + | "resources" + | "metadata"; + +export const BlueprintCreatePage = ({ + onBack, + onCreate, + baseBlueprintId, +}: BlueprintCreatePageProps) => { + const [screenState, setScreenState] = React.useState( + baseBlueprintId ? "loading-base" : "form", + ); + const [formData, setFormData] = React.useState({ + name: "", + sourceType: baseBlueprintId ? "base_blueprint" : "dockerfile", + dockerfile: "", + baseBlueprintId: baseBlueprintId || "", + systemSetupCommands: [], + architecture: "", + resources: "", + metadata: {}, + }); + const [activeFieldIndex, setActiveFieldIndex] = React.useState(0); + const [error, setError] = React.useState(null); + const [validationError, setValidationError] = React.useState( + null, + ); + const [setupExpanded, setSetupExpanded] = React.useState(false); + const [inMetadataSection, setInMetadataSection] = React.useState(false); + const [metadataKey, setMetadataKey] = React.useState(""); + const [metadataValue, setMetadataValue] = React.useState(""); + const [metadataInputMode, setMetadataInputMode] = React.useState< + "key" | "value" | null + >(null); + const [selectedMetadataIndex, setSelectedMetadataIndex] = React.useState(0); + + useExitOnCtrlC(); + + // Load base blueprint for duplication + React.useEffect(() => { + if (!baseBlueprintId) return; + + getBlueprint(baseBlueprintId) + .then((bp) => { + const lp = bp.parameters?.launch_parameters; + const params = bp.parameters; + setFormData({ + name: (bp.name || "blueprint") + "-copy", + sourceType: "base_blueprint", + dockerfile: params?.dockerfile || "", + baseBlueprintId: bp.id, + systemSetupCommands: params?.system_setup_commands || [], + architecture: lp?.architecture || "", + resources: lp?.resource_size_request || "", + metadata: (bp.metadata as Record) || {}, + }); + setScreenState("form"); + }) + .catch((err) => { + setError(err as Error); + setScreenState("error"); + }); + }, [baseBlueprintId]); + + const visibleFields = React.useMemo((): Array<{ + key: FieldKey; + label: string; + type: "text" | "select" | "action" | "list" | "metadata"; + }> => { + const f: Array<{ + key: FieldKey; + label: string; + type: "text" | "select" | "action" | "list" | "metadata"; + }> = [ + { key: "submit", label: "Create Blueprint", type: "action" }, + { key: "name", label: "Name (required)", type: "text" }, + { key: "sourceType", label: "Source", type: "select" }, + ]; + if (formData.sourceType === "dockerfile") { + f.push({ key: "dockerfile", label: "Dockerfile", type: "text" }); + } else { + f.push({ + key: "baseBlueprintId", + label: "Base Blueprint ID", + type: "text", + }); + } + f.push({ + key: "setupCommands", + label: "System Setup Commands", + type: "list", + }); + f.push({ key: "architecture", label: "Architecture", type: "select" }); + f.push({ key: "resources", label: "Resources", type: "select" }); + f.push({ key: "metadata", label: "Metadata (optional)", type: "metadata" }); + return f; + }, [formData.sourceType]); + + const activeField = visibleFields[activeFieldIndex]?.key; + + const handleSourceTypeSelect = useFormSelectNavigation( + formData.sourceType, + SOURCE_TYPE_OPTIONS, + (v) => setFormData((prev) => ({ ...prev, sourceType: v as SourceType })), + activeField === "sourceType", + ); + + const handleArchitectureSelect = useFormSelectNavigation( + formData.architecture || "x86_64", + ARCHITECTURE_OPTIONS, + (v) => setFormData((prev) => ({ ...prev, architecture: v })), + activeField === "architecture", + ); + + const handleResourcesSelect = useFormSelectNavigation( + formData.resources || "SMALL", + RESOURCE_OPTIONS, + (v) => setFormData((prev) => ({ ...prev, resources: v })), + activeField === "resources", + ); + + const handleSubmit = async () => { + if (!formData.name.trim()) { + setValidationError("Name is required"); + const idx = visibleFields.findIndex((f) => f.key === "name"); + if (idx >= 0) setActiveFieldIndex(idx); + return; + } + + setError(null); + setValidationError(null); + setScreenState("creating"); + + try { + const bp = await createBlueprint({ + name: formData.name.trim(), + dockerfile: + formData.sourceType === "dockerfile" && formData.dockerfile.trim() + ? formData.dockerfile.trim() + : undefined, + baseBlueprintId: + formData.sourceType === "base_blueprint" && + formData.baseBlueprintId.trim() + ? formData.baseBlueprintId.trim() + : undefined, + systemSetupCommands: formData.systemSetupCommands.filter( + (c) => c.trim().length > 0, + ), + architecture: formData.architecture || undefined, + resourceSizeRequest: formData.resources || undefined, + metadata: + Object.keys(formData.metadata).length > 0 + ? formData.metadata + : undefined, + }); + onCreate(bp.id); + } catch (err) { + setError(err as Error); + setScreenState("error"); + } + }; + + // Main form input + useInput( + (input, key) => { + if (screenState === "error") { + if (input === "r" || key.return) { + setError(null); + setScreenState("form"); + } else if (input === "q" || key.escape) { + onBack(); + } + return; + } + + if (screenState !== "form") return; + + if (handleSourceTypeSelect(input, key)) return; + if (handleArchitectureSelect(input, key)) return; + if (handleResourcesSelect(input, key)) return; + + if (input === "q" || key.escape) { + onBack(); + return; + } + + if (input === "s" && key.ctrl) { + handleSubmit(); + return; + } + + if (activeField === "setupCommands" && key.return) { + setSetupExpanded(true); + return; + } + + if (activeField === "metadata" && key.return) { + setInMetadataSection(true); + setSelectedMetadataIndex(0); + return; + } + + if (key.return) { + handleSubmit(); + return; + } + + if (key.upArrow || (key.tab && key.shift)) { + setActiveFieldIndex((prev) => Math.max(0, prev - 1)); + return; + } + + if (key.downArrow || (key.tab && !key.shift)) { + setActiveFieldIndex((prev) => + Math.min(visibleFields.length - 1, prev + 1), + ); + return; + } + }, + { isActive: !setupExpanded && !inMetadataSection }, + ); + + // Metadata section input + useInput( + (input, key) => { + const metadataKeys = Object.keys(formData.metadata); + const maxIndex = metadataKeys.length + 1; + + if (metadataInputMode) { + if (metadataInputMode === "key" && key.return && metadataKey.trim()) { + setMetadataInputMode("value"); + return; + } else if (metadataInputMode === "value" && key.return) { + if (metadataKey.trim() && metadataValue.trim()) { + setFormData((prev) => ({ + ...prev, + metadata: { + ...prev.metadata, + [metadataKey.trim()]: metadataValue.trim(), + }, + })); + } + setMetadataKey(""); + setMetadataValue(""); + setMetadataInputMode(null); + setSelectedMetadataIndex(0); + return; + } else if (key.escape) { + setMetadataKey(""); + setMetadataValue(""); + setMetadataInputMode(null); + return; + } else if (key.tab) { + setMetadataInputMode(metadataInputMode === "key" ? "value" : "key"); + return; + } + return; + } + + if (key.upArrow && selectedMetadataIndex > 0) { + setSelectedMetadataIndex(selectedMetadataIndex - 1); + } else if (key.downArrow && selectedMetadataIndex < maxIndex) { + setSelectedMetadataIndex(selectedMetadataIndex + 1); + } else if (key.return) { + if (selectedMetadataIndex === 0) { + setMetadataKey(""); + setMetadataValue(""); + setMetadataInputMode("key"); + } else if (selectedMetadataIndex === maxIndex) { + setInMetadataSection(false); + setSelectedMetadataIndex(0); + } else { + const keyToEdit = metadataKeys[selectedMetadataIndex - 1]; + setMetadataKey(keyToEdit || ""); + setMetadataValue(formData.metadata[keyToEdit] || ""); + const newMetadata = { ...formData.metadata }; + delete newMetadata[keyToEdit]; + setFormData((prev) => ({ ...prev, metadata: newMetadata })); + setMetadataInputMode("key"); + } + } else if ( + (input === "d" || key.delete) && + selectedMetadataIndex >= 1 && + selectedMetadataIndex <= metadataKeys.length + ) { + const keyToDelete = metadataKeys[selectedMetadataIndex - 1]; + const newMetadata = { ...formData.metadata }; + delete newMetadata[keyToDelete]; + setFormData((prev) => ({ ...prev, metadata: newMetadata })); + const newLength = Object.keys(newMetadata).length; + if (selectedMetadataIndex > newLength) { + setSelectedMetadataIndex(Math.max(0, newLength)); + } + } else if (key.escape || input === "q") { + setInMetadataSection(false); + setSelectedMetadataIndex(0); + setMetadataKey(""); + setMetadataValue(""); + setMetadataInputMode(null); + } + }, + { isActive: inMetadataSection }, + ); + + const breadcrumbItems = [ + { label: "Blueprints" }, + { + label: baseBlueprintId ? "Duplicate" : "Create", + active: true, + }, + ]; + + if (screenState === "loading-base") { + return ( + <> + + + + ); + } + + if (screenState === "creating") { + return ( + <> + + + + ); + } + + if (screenState === "error") { + return ( + <> + + + + + ); + } + + // Render metadata section + const renderMetadata = (isActive: boolean) => { + if (!inMetadataSection) { + return ( + + + + {isActive ? figures.pointer : " "} Metadata (optional):{" "} + + + {Object.keys(formData.metadata).length > 0 + ? `${Object.keys(formData.metadata).length} item(s)` + : "None"} + + {isActive && ( + + {" "} + [Enter to manage] + + )} + + {Object.keys(formData.metadata).length > 0 && ( + + + + )} + + ); + } + + const metadataKeys = Object.keys(formData.metadata); + const maxIndex = metadataKeys.length + 1; + + return ( + + + {figures.hamburger} Manage Metadata + + + {metadataInputMode && ( + + + {selectedMetadataIndex === 0 ? "Adding New" : "Editing"} + + + {metadataInputMode === "key" ? ( + <> + Key: + + + ) : ( + Key: {metadataKey || ""} + )} + + + {metadataInputMode === "value" ? ( + <> + Value: + + + ) : ( + Value: {metadataValue || ""} + )} + + + )} + + {!metadataInputMode && ( + <> + + + {selectedMetadataIndex === 0 ? figures.pointer : " "}{" "} + + + + Add new metadata + + + + {metadataKeys.length > 0 && ( + + {metadataKeys.map((key, index) => { + const itemIndex = index + 1; + const isSelected = selectedMetadataIndex === itemIndex; + return ( + + + {isSelected ? figures.pointer : " "}{" "} + + + {key}: {formData.metadata[key]} + + + ); + })} + + )} + + + + {selectedMetadataIndex === maxIndex + ? figures.pointer + : " "}{" "} + + + {figures.tick} Done + + + + )} + + + + {metadataInputMode + ? `[Tab] Switch field • [Enter] ${metadataInputMode === "key" ? "Next" : "Save"} • [esc] Cancel` + : `${figures.arrowUp}${figures.arrowDown} Navigate • [Enter] ${selectedMetadataIndex === 0 ? "Add" : selectedMetadataIndex === maxIndex ? "Done" : "Edit"} • [d] Delete • [esc] Back`} + + + + ); + }; + + return ( + <> + + + + + {figures.info} Note:{" "} + {baseBlueprintId + ? "Duplicating blueprint. Modify fields as needed." + : "Create a new blueprint. Provide a Dockerfile or base blueprint."} + + + + + {visibleFields.map((field, idx) => { + const isActive = activeFieldIndex === idx; + + if (field.type === "action") { + return ( + + ); + } + + if (field.type === "text") { + let value = ""; + let placeholder = ""; + if (field.key === "name") { + value = formData.name; + placeholder = "my-blueprint"; + } else if (field.key === "dockerfile") { + value = formData.dockerfile; + placeholder = "FROM ubuntu:22.04"; + } else if (field.key === "baseBlueprintId") { + value = formData.baseBlueprintId; + placeholder = "bpt_... or blueprint-name"; + } + + const hasError = + field.key === "name" && validationError === "Name is required"; + + return ( + { + if (field.key === "name") { + setFormData((prev) => ({ ...prev, name: newValue })); + } else if (field.key === "dockerfile") { + setFormData((prev) => ({ ...prev, dockerfile: newValue })); + } else if (field.key === "baseBlueprintId") { + setFormData((prev) => ({ + ...prev, + baseBlueprintId: newValue, + })); + } + if (validationError) setValidationError(null); + }} + onSubmit={handleSubmit} + isActive={isActive} + placeholder={placeholder} + error={hasError ? validationError : undefined} + /> + ); + } + + if (field.type === "select") { + if (field.key === "sourceType") { + return ( + + setFormData((prev) => ({ + ...prev, + sourceType: v as SourceType, + })) + } + isActive={isActive} + /> + ); + } + if (field.key === "architecture") { + return ( + + setFormData((prev) => ({ ...prev, architecture: v })) + } + isActive={isActive} + /> + ); + } + if (field.key === "resources") { + return ( + + setFormData((prev) => ({ ...prev, resources: v })) + } + isActive={isActive} + /> + ); + } + } + + if (field.type === "list") { + return ( + + setFormData((prev) => ({ + ...prev, + systemSetupCommands: items, + })) + } + isActive={isActive} + isExpanded={setupExpanded} + onExpandedChange={setSetupExpanded} + itemPlaceholder="apt-get install -y ..." + addLabel="+ Add command" + collapsedLabel="command(s)" + /> + ); + } + + if (field.type === "metadata") { + return ( + + {renderMetadata(isActive)} + + ); + } + + return null; + })} + + + {!setupExpanded && !inMetadataSection && ( + + )} + + ); +}; diff --git a/src/router/Router.tsx b/src/router/Router.tsx index 5c5248d..0971df0 100644 --- a/src/router/Router.tsx +++ b/src/router/Router.tsx @@ -32,6 +32,7 @@ const KNOWN_SCREENS: Set = new Set([ "snapshot-detail", "blueprint-list", "blueprint-detail", + "blueprint-create", "blueprint-logs", "network-policy-list", "network-policy-detail", @@ -51,6 +52,8 @@ const KNOWN_SCREENS: Set = new Set([ "agent-create", "axon-list", "axon-detail", + "axon-events", + "axon-sql", "object-list", "object-detail", "object-create", @@ -60,6 +63,7 @@ const KNOWN_SCREENS: Set = new Set([ "benchmark-detail", "benchmark-run-list", "benchmark-run-detail", + "scenario-create", "scenario-run-list", "scenario-run-detail", "benchmark-job-list", @@ -116,6 +120,7 @@ import { SnapshotListScreen } from "../screens/SnapshotListScreen.js"; import { SnapshotDetailScreen } from "../screens/SnapshotDetailScreen.js"; import { BlueprintListScreen } from "../screens/BlueprintListScreen.js"; import { BlueprintDetailScreen } from "../screens/BlueprintDetailScreen.js"; +import { BlueprintCreateScreen } from "../screens/BlueprintCreateScreen.js"; import { BlueprintLogsScreen } from "../screens/BlueprintLogsScreen.js"; import { NetworkPolicyListScreen } from "../screens/NetworkPolicyListScreen.js"; import { NetworkPolicyDetailScreen } from "../screens/NetworkPolicyDetailScreen.js"; @@ -134,6 +139,8 @@ import { AgentDetailScreen } from "../screens/AgentDetailScreen.js"; import { AgentCreateScreen } from "../screens/AgentCreateScreen.js"; import { AxonListScreen } from "../screens/AxonListScreen.js"; import { AxonDetailScreen } from "../screens/AxonDetailScreen.js"; +import { AxonEventsScreen } from "../screens/AxonEventsScreen.js"; +import { AxonSqlScreen } from "../screens/AxonSqlScreen.js"; import { ObjectListScreen } from "../screens/ObjectListScreen.js"; import { ObjectDetailScreen } from "../screens/ObjectDetailScreen.js"; import { ObjectCreateScreen } from "../screens/ObjectCreateScreen.js"; @@ -143,6 +150,7 @@ import { BenchmarkListScreen } from "../screens/BenchmarkListScreen.js"; import { BenchmarkDetailScreen } from "../screens/BenchmarkDetailScreen.js"; import { BenchmarkRunListScreen } from "../screens/BenchmarkRunListScreen.js"; import { BenchmarkRunDetailScreen } from "../screens/BenchmarkRunDetailScreen.js"; +import { ScenarioCreateScreen } from "../screens/ScenarioCreateScreen.js"; import { ScenarioRunListScreen } from "../screens/ScenarioRunListScreen.js"; import { ScenarioRunDetailScreen } from "../screens/ScenarioRunDetailScreen.js"; import { BenchmarkJobListScreen } from "../screens/BenchmarkJobListScreen.js"; @@ -181,6 +189,7 @@ export function Router() { case "blueprint-list": case "blueprint-detail": + case "blueprint-create": case "blueprint-logs": if (!currentScreen.startsWith("blueprint")) { useBlueprintStore.getState().clearAll(); @@ -231,6 +240,7 @@ export function Router() { case "benchmark-detail": case "benchmark-run-list": case "benchmark-run-detail": + case "scenario-create": case "scenario-run-list": case "scenario-run-detail": if ( @@ -252,6 +262,8 @@ export function Router() { case "agent-list": case "agent-detail": case "agent-create": + case "axon-events": + case "axon-sql": break; } } @@ -298,6 +310,12 @@ export function Router() { {currentScreen === "blueprint-detail" && ( )} + {currentScreen === "blueprint-create" && ( + + )} {currentScreen === "blueprint-logs" && ( )} @@ -349,6 +367,12 @@ export function Router() { {currentScreen === "axon-detail" && ( )} + {currentScreen === "axon-events" && ( + + )} + {currentScreen === "axon-sql" && ( + + )} {currentScreen === "object-list" && ( )} @@ -376,6 +400,9 @@ export function Router() { {currentScreen === "benchmark-run-detail" && ( )} + {currentScreen === "scenario-create" && ( + + )} {currentScreen === "scenario-run-list" && ( )} diff --git a/src/screens/AxonDetailScreen.tsx b/src/screens/AxonDetailScreen.tsx index ccc6c36..8b9bbac 100644 --- a/src/screens/AxonDetailScreen.tsx +++ b/src/screens/AxonDetailScreen.tsx @@ -23,7 +23,7 @@ interface AxonDetailScreenProps { } export function AxonDetailScreen({ axonId }: AxonDetailScreenProps) { - const { goBack } = useNavigation(); + const { goBack, navigate } = useNavigation(); const { data: axon, error } = useResourceDetail({ id: axonId, @@ -147,8 +147,32 @@ export function AxonDetailScreen({ axonId }: AxonDetailScreenProps) { getUrl={(a) => getAxonUrl(a.id)} getStatus={() => "active"} detailSections={detailSections} - operations={[]} - onOperation={async () => {}} + operations={[ + { + key: "events", + label: "View Events", + color: colors.info, + icon: figures.info, + shortcut: "e", + }, + { + key: "sql", + label: "SQL Workbench", + color: colors.warning, + icon: figures.pointer, + shortcut: "s", + }, + ]} + onOperation={async (operation: string) => { + switch (operation) { + case "events": + navigate("axon-events", { axonId: axon.id }); + break; + case "sql": + navigate("axon-sql", { axonId: axon.id }); + break; + } + }} onBack={goBack} buildDetailLines={buildDetailLines} /> diff --git a/src/screens/AxonEventsScreen.tsx b/src/screens/AxonEventsScreen.tsx new file mode 100644 index 0000000..6bb5edb --- /dev/null +++ b/src/screens/AxonEventsScreen.tsx @@ -0,0 +1,236 @@ +import React from "react"; +import { Box, Text, useInput } from "ink"; +import figures from "figures"; +import { useNavigation } from "../store/navigationStore.js"; +import { + listAxonEvents, + getAxon, + type AxonEvent, +} from "../services/axonService.js"; +import { Table, createTextColumn } from "../components/Table.js"; +import { SpinnerComponent } from "../components/Spinner.js"; +import { ErrorMessage } from "../components/ErrorMessage.js"; +import { Breadcrumb } from "../components/Breadcrumb.js"; +import { NavigationTips } from "../components/NavigationTips.js"; +import { colors } from "../utils/theme.js"; +import { formatTimeAgo } from "../components/ResourceListView.js"; +import { useExitOnCtrlC } from "../hooks/useExitOnCtrlC.js"; + +interface AxonEventsScreenProps { + axonId?: string; +} + +const PAGE_SIZE = 20; + +export function AxonEventsScreen({ axonId }: AxonEventsScreenProps) { + const { goBack, params } = useNavigation(); + const id = axonId || (params.axonId as string); + + const [events, setEvents] = React.useState([]); + const [loading, setLoading] = React.useState(true); + const [error, setError] = React.useState(null); + const [offset, setOffset] = React.useState(0); + const [selectedIndex, setSelectedIndex] = React.useState(0); + const [hasMore, setHasMore] = React.useState(false); + const [axonName, setAxonName] = React.useState(""); + const [refreshTrigger, setRefreshTrigger] = React.useState(0); + + useExitOnCtrlC(); + + // Fetch axon name on mount + React.useEffect(() => { + if (!id) return; + + let cancelled = false; + + const fetchAxonName = async () => { + try { + const axon = await getAxon(id); + if (!cancelled) { + setAxonName(axon.name || id); + } + } catch { + // Silently fail, fallback to ID + if (!cancelled) { + setAxonName(id); + } + } + }; + + fetchAxonName(); + + return () => { + cancelled = true; + }; + }, [id]); + + React.useEffect(() => { + if (!id) { + goBack(); + } + }, [id, goBack]); + + // Fetch events + React.useEffect(() => { + if (!id) return; + + let cancelled = false; + + const fetchEvents = async () => { + try { + setLoading(true); + setError(null); + const result = await listAxonEvents(id, { limit: PAGE_SIZE, offset }); + if (!cancelled) { + setEvents(result.events); + setHasMore(result.hasMore); + setLoading(false); + setSelectedIndex(0); + } + } catch (err) { + if (!cancelled) { + setError(err as Error); + setLoading(false); + } + } + }; + + fetchEvents(); + + return () => { + cancelled = true; + }; + }, [id, offset, refreshTrigger]); + + useInput((input, key) => { + if (loading) return; + + if (input === "q" || key.escape) { + goBack(); + } else if (input === "n" && hasMore) { + setOffset((prev) => prev + PAGE_SIZE); + } else if (input === "p" && offset > 0) { + setOffset((prev) => Math.max(0, prev - PAGE_SIZE)); + } else if (input === "r") { + setRefreshTrigger((prev) => prev + 1); + } else if (key.upArrow) { + setSelectedIndex((prev) => Math.max(0, prev - 1)); + } else if (key.downArrow) { + setSelectedIndex((prev) => Math.min(events.length - 1, prev + 1)); + } + }); + + if (!id) { + return null; + } + + const breadcrumbItems = [ + { label: "Axons" }, + { label: axonName || id }, + { label: "Events", active: true }, + ]; + + if (loading) { + return ( + <> + + + + ); + } + + if (error) { + return ( + <> + + + + + Press [r] to retry or [q] to go back + + + + ); + } + + const columns = [ + createTextColumn( + "sequence", + "SEQ", + (event) => String(event.sequence), + { width: 8 }, + ), + createTextColumn( + "timestamp", + "TIME", + (event) => formatTimeAgo(event.timestamp_ms), + { width: 22 }, + ), + createTextColumn("origin", "ORIGIN", (event) => event.origin, { + width: 16, + }), + createTextColumn("source", "SOURCE", (event) => event.source, { + width: 14, + }), + createTextColumn( + "event_type", + "TYPE", + (event) => event.event_type, + { width: 18 }, + ), + createTextColumn( + "payload", + "PAYLOAD", + (event) => event.payload, + { width: 30 }, + ), + ]; + + const pageNumber = Math.floor(offset / PAGE_SIZE) + 1; + + return ( + <> + + + + + {figures.info} Axon Events + + + {" "} + - Page {pageNumber} + + + + String(event.sequence)} + emptyState={ + + {figures.info} No events found + + } + /> + + + + Showing {events.length} events + + + + + 0 ? { key: "p", label: "Previous page" } : null, + { key: "r", label: "Refresh" }, + { key: "q", label: "Back" }, + ].filter((tip): tip is { key: string; label: string } => tip !== null)} + /> + + ); +} diff --git a/src/screens/AxonSqlScreen.tsx b/src/screens/AxonSqlScreen.tsx new file mode 100644 index 0000000..d216fef --- /dev/null +++ b/src/screens/AxonSqlScreen.tsx @@ -0,0 +1,311 @@ +import React from "react"; +import { Box, Text, useInput } from "ink"; +import TextInput from "ink-text-input"; +import figures from "figures"; +import { useNavigation } from "../store/navigationStore.js"; +import { + executeAxonSql, + getAxon, + type SqlQueryResultView, +} from "../services/axonService.js"; +import { Table, createTextColumn } from "../components/Table.js"; +import { SpinnerComponent } from "../components/Spinner.js"; +import { ErrorMessage } from "../components/ErrorMessage.js"; +import { Breadcrumb } from "../components/Breadcrumb.js"; +import { NavigationTips } from "../components/NavigationTips.js"; +import { colors } from "../utils/theme.js"; +import { useExitOnCtrlC } from "../hooks/useExitOnCtrlC.js"; + +interface AxonSqlScreenProps { + axonId?: string; +} + +type ScreenState = "input" | "executing" | "results" | "error"; + +const DEFAULT_QUERY = + "SELECT sequence, timestamp_ms, source, event_type, origin, payload FROM rl_axon_events LIMIT 5;"; + +export function AxonSqlScreen({ axonId }: AxonSqlScreenProps) { + const { goBack, params } = useNavigation(); + const id = axonId || (params.axonId as string); + + const [screenState, setScreenState] = React.useState("input"); + const [query, setQuery] = React.useState(DEFAULT_QUERY); + const [result, setResult] = React.useState(null); + const [error, setError] = React.useState(null); + const [axonName, setAxonName] = React.useState(""); + const [executedQuery, setExecutedQuery] = React.useState(""); + + useExitOnCtrlC(); + + // Fetch axon name on mount + React.useEffect(() => { + if (!id) return; + + let cancelled = false; + + const fetchAxonName = async () => { + try { + const axon = await getAxon(id); + if (!cancelled) { + setAxonName(axon.name || id); + } + } catch { + // Silently fail, fallback to ID + if (!cancelled) { + setAxonName(id); + } + } + }; + + fetchAxonName(); + + return () => { + cancelled = true; + }; + }, [id]); + + const executeQuery = React.useCallback(async () => { + if (!id || !query.trim()) return; + + setScreenState("executing"); + setExecutedQuery(query); + + try { + const queryResult = await executeAxonSql(id, query); + setResult(queryResult); + setError(null); + setScreenState("results"); + } catch (err) { + setError(err as Error); + setResult(null); + setScreenState("error"); + } + }, [id, query]); + + useInput((input, key) => { + if (screenState === "input") { + if (key.escape) { + goBack(); + } else if (key.return) { + executeQuery(); + } + } else if (screenState === "results") { + if (key.escape || input === "q") { + goBack(); + } else if (input === "e") { + setScreenState("input"); + } else if (input === "r") { + executeQuery(); + } + } else if (screenState === "error") { + if (key.escape || input === "q") { + goBack(); + } else if (input === "e") { + setScreenState("input"); + } else if (input === "r") { + executeQuery(); + } + } + }); + + React.useEffect(() => { + if (!id) { + goBack(); + } + }, [id, goBack]); + + if (!id) { + return null; + } + + const breadcrumbItems = [ + { label: "Axons" }, + { label: axonName || id }, + { label: "SQL Workbench", active: true }, + ]; + + // Input mode + if (screenState === "input") { + return ( + <> + + + + + {figures.pointer} SQL Workbench + + + + + SQL Query: + + + + + + + + + Press [Enter] to execute or [Esc] to go back + + + + + + + ); + } + + // Executing mode + if (screenState === "executing") { + return ( + <> + + + + ); + } + + // Error mode + if (screenState === "error") { + return ( + <> + + + + + {figures.cross} Query Failed + + + + + + Query: {executedQuery} + + + + + + + + Press [e] to edit query, [r] to retry, or [q] to go back + + + + + + + ); + } + + // Results mode + if (screenState === "results" && result) { + const normalizedRows = result.rows.map((row, rowIndex) => { + const obj: Record = Array.isArray(row) + ? Object.fromEntries( + result.columns.map((col, i) => [col.name, (row as unknown[])[i]]), + ) + : { ...(row as Record) }; + obj.__rowIndex = rowIndex; + return obj; + }); + + // Build dynamic columns from result.columns + const tableColumns = result.columns.map((col) => + createTextColumn>( + col.name, + col.name + (col.type ? ` (${col.type})` : ""), + (row) => { + const val = row[col.name]; + if (val === null || val === undefined) return "NULL"; + return String(val); + }, + { width: Math.max(col.name.length + (col.type?.length ?? 0) + 4, 15) }, + ), + ); + + return ( + <> + + + + + {figures.tick} Query Results + + + + + + Query: {executedQuery} + + + + + + {figures.info} {result.meta.duration_ms}ms + + + {" "} + • {normalizedRows.length} rows + + {result.meta.changes > 0 && ( + + {" "} + • {result.meta.changes} changes + + )} + {result.meta.rows_read_limit_reached && ( + + {" "} + • Results truncated (limit reached) + + )} + + +
String(row.__rowIndex)} + emptyState={ + + {figures.info} No results + + } + /> + + + + + ); + } + + return null; +} diff --git a/src/screens/BenchmarkDetailScreen.tsx b/src/screens/BenchmarkDetailScreen.tsx index 71f24bb..d09c7b1 100644 --- a/src/screens/BenchmarkDetailScreen.tsx +++ b/src/screens/BenchmarkDetailScreen.tsx @@ -175,6 +175,13 @@ export function BenchmarkDetailScreen({ // Operations available for benchmarks const operations: ResourceOperation[] = [ + { + key: "view-runs", + label: "View Benchmark Runs", + color: colors.info, + icon: figures.arrowRight, + shortcut: "r", + }, { key: "create-job", label: "Create Benchmark Job", @@ -187,6 +194,9 @@ export function BenchmarkDetailScreen({ // Handle operation selection const handleOperation = async (operation: string, resource: Benchmark) => { switch (operation) { + case "view-runs": + navigate("benchmark-run-list", { benchmarkId: resource.id }); + break; case "create-job": navigate("benchmark-job-create", { initialBenchmarkIds: resource.id }); break; diff --git a/src/screens/BenchmarkRunDetailScreen.tsx b/src/screens/BenchmarkRunDetailScreen.tsx index 0c6c05e..6cccba8 100644 --- a/src/screens/BenchmarkRunDetailScreen.tsx +++ b/src/screens/BenchmarkRunDetailScreen.tsx @@ -20,7 +20,10 @@ import { import { getBenchmarkRun, listScenarioRuns, + cancelBenchmarkRun, + completeBenchmarkRun, } from "../services/benchmarkService.js"; +import { ConfirmationPrompt } from "../components/ConfirmationPrompt.js"; import { useResourceDetail } from "../hooks/useResourceDetail.js"; import { SpinnerComponent } from "../components/Spinner.js"; import { ErrorMessage } from "../components/ErrorMessage.js"; @@ -60,6 +63,13 @@ export function BenchmarkRunDetailScreen({ const [scenarioRuns, setScenarioRuns] = React.useState([]); const [scenarioRunsLoading, setScenarioRunsLoading] = React.useState(false); + const [showCancelConfirm, setShowCancelConfirm] = React.useState(false); + const [showCompleteConfirm, setShowCompleteConfirm] = React.useState(false); + const [operating, setOperating] = React.useState(false); + const [operationError, setOperationError] = React.useState( + null, + ); + const displayError = error ?? operationError; // Fetch scenario runs for this benchmark run React.useEffect(() => { @@ -104,6 +114,94 @@ export function BenchmarkRunDetailScreen({ return () => clearInterval(interval); }, [benchmarkRunId, run]); + // Execute cancel after confirmation + const executeCancel = async () => { + if (!run) return; + setShowCancelConfirm(false); + setOperating(true); + try { + await cancelBenchmarkRun(run.id); + } catch (err) { + setOperationError(err as Error); + } + setOperating(false); + }; + + // Execute complete after confirmation + const executeComplete = async () => { + if (!run) return; + setShowCompleteConfirm(false); + setOperating(true); + try { + await completeBenchmarkRun(run.id); + } catch (err) { + setOperationError(err as Error); + } + setOperating(false); + }; + + // Show cancel confirmation + if (showCancelConfirm && run) { + return ( + setShowCancelConfirm(false)} + /> + ); + } + + // Show complete confirmation + if (showCompleteConfirm && run) { + return ( + setShowCompleteConfirm(false)} + /> + ); + } + + // Show operating state + if (operating) { + return ( + <> + + + + ); + } + // Show loading state if (!run && benchmarkRunId && !error) { return ( @@ -122,7 +220,7 @@ export function BenchmarkRunDetailScreen({ } // Show error state - if (error && !run) { + if (displayError && !run) { return ( <> ); @@ -449,6 +547,21 @@ export function BenchmarkRunDetailScreen({ }); } + if (operationError) { + detailSections.push({ + title: "Operation Error", + icon: figures.cross, + color: colors.error, + fields: [ + { + label: "Error", + value: operationError.message, + color: colors.error, + }, + ], + }); + } + // Operations available for benchmark runs const operations: ResourceOperation[] = [ { @@ -460,12 +573,37 @@ export function BenchmarkRunDetailScreen({ }, ]; + if (run.state === "running") { + operations.push( + { + key: "cancel", + label: "Cancel Run", + color: colors.error, + icon: figures.cross, + shortcut: "x", + }, + { + key: "complete", + label: "Complete Run", + color: colors.success, + icon: figures.tick, + shortcut: "f", + }, + ); + } + // Handle operation selection const handleOperation = async (operation: string, resource: BenchmarkRun) => { switch (operation) { case "view-scenarios": navigate("scenario-run-list", { benchmarkRunId: resource.id }); break; + case "cancel": + setShowCancelConfirm(true); + break; + case "complete": + setShowCompleteConfirm(true); + break; } }; diff --git a/src/screens/BenchmarkRunListScreen.tsx b/src/screens/BenchmarkRunListScreen.tsx index 38c92c8..181298b 100644 --- a/src/screens/BenchmarkRunListScreen.tsx +++ b/src/screens/BenchmarkRunListScreen.tsx @@ -29,7 +29,13 @@ import type { BenchmarkRun } from "../store/benchmarkStore.js"; import { openInBrowser } from "../utils/browser.js"; import { getBenchmarkRunUrl } from "../utils/url.js"; -export function BenchmarkRunListScreen() { +interface BenchmarkRunListScreenProps { + benchmarkId?: string; +} + +export function BenchmarkRunListScreen({ + benchmarkId, +}: BenchmarkRunListScreenProps) { const { exit: inkExit } = useApp(); const { navigate, goBack } = useNavigation(); const [selectedIndex, setSelectedIndex] = React.useState(0); @@ -71,6 +77,7 @@ export function BenchmarkRunListScreen() { limit: params.limit, startingAfter: params.startingAt, includeTotalCount: params.includeTotalCount, + benchmarkId, }); return { @@ -79,7 +86,7 @@ export function BenchmarkRunListScreen() { totalCount: result.totalCount, }; }, - [], + [benchmarkId], ); // Use the shared pagination hook @@ -101,7 +108,7 @@ export function BenchmarkRunListScreen() { getItemId: (run: BenchmarkRun) => run.id, pollInterval: 5000, pollingEnabled: !showPopup && !search.searchMode, - deps: [PAGE_SIZE], + deps: [PAGE_SIZE, benchmarkId], }); // Operations for benchmark runs @@ -291,17 +298,19 @@ export function BenchmarkRunListScreen() { } }); + const breadcrumbItems = [ + { label: "Home" }, + { label: "Benchmarks" }, + ...(benchmarkId + ? [{ label: benchmarkId }, { label: "Runs", active: true }] + : [{ label: "Benchmark Runs", active: true }]), + ]; + // Loading state if (loading && benchmarkRuns.length === 0) { return ( <> - + ); @@ -311,13 +320,7 @@ export function BenchmarkRunListScreen() { if (error) { return ( <> - + ); @@ -326,13 +329,7 @@ export function BenchmarkRunListScreen() { // Main list view return ( <> - + {/* Search bar */} navigate("blueprint-detail", { blueprintId })} + baseBlueprintId={baseBlueprintId} + /> + ); +} diff --git a/src/screens/BlueprintDetailScreen.tsx b/src/screens/BlueprintDetailScreen.tsx index 317807b..d521b82 100644 --- a/src/screens/BlueprintDetailScreen.tsx +++ b/src/screens/BlueprintDetailScreen.tsx @@ -266,6 +266,13 @@ export function BlueprintDetailScreen({ icon: figures.play, shortcut: "c", }, + { + key: "duplicate", + label: "Duplicate Blueprint", + color: colors.secondary, + icon: figures.ellipsis, + shortcut: "u", + }, { key: "delete", label: "Delete Blueprint", @@ -284,6 +291,9 @@ export function BlueprintDetailScreen({ case "create-devbox": navigate("devbox-create", { blueprintId: resource.id }); break; + case "duplicate": + navigate("blueprint-create", { baseBlueprintId: resource.id }); + break; case "delete": // Show confirmation dialog setShowDeleteConfirm(true); diff --git a/src/screens/ScenarioCreateScreen.tsx b/src/screens/ScenarioCreateScreen.tsx new file mode 100644 index 0000000..f94f94d --- /dev/null +++ b/src/screens/ScenarioCreateScreen.tsx @@ -0,0 +1,1232 @@ +import React from "react"; +import { Box, Text, useInput } from "ink"; +import TextInput from "ink-text-input"; +import figures from "figures"; +import { SpinnerComponent } from "../components/Spinner.js"; +import { ErrorMessage } from "../components/ErrorMessage.js"; +import { Breadcrumb } from "../components/Breadcrumb.js"; +import { NavigationTips } from "../components/NavigationTips.js"; +import { MetadataDisplay } from "../components/MetadataDisplay.js"; +import { + FormTextInput, + FormSelect, + FormActionButton, + FormListManager, + useFormSelectNavigation, +} from "../components/form/index.js"; +import { colors } from "../utils/theme.js"; +import { useExitOnCtrlC } from "../hooks/useExitOnCtrlC.js"; +import { useNavigation } from "../store/navigationStore.js"; +import { createScenario } from "../services/scenarioService.js"; +import type { ScenarioCreateParams } from "@runloop/api-client/resources/scenarios/scenarios"; + +type ScreenState = "form" | "editing-scorer" | "creating" | "error"; +type ScorerType = + | "command_scorer" + | "bash_script_scorer" + | "python_script_scorer" + | "test_based_scorer" + | "ast_grep_scorer" + | "custom_scorer"; +type EnvironmentSource = "none" | "blueprint" | "snapshot"; +type ValidationType = "UNSPECIFIED" | "FORWARD" | "REVERSE" | "EVALUATION"; + +const SCORER_TYPE_OPTIONS = [ + "command_scorer", + "bash_script_scorer", + "python_script_scorer", + "test_based_scorer", + "ast_grep_scorer", + "custom_scorer", +] as const; + +const ENVIRONMENT_OPTIONS = ["none", "blueprint", "snapshot"] as const; +const VALIDATION_OPTIONS = [ + "UNSPECIFIED", + "FORWARD", + "REVERSE", + "EVALUATION", +] as const; + +interface ScorerFormData { + name: string; + weight: string; + type: ScorerType; + command: string; + bashScript: string; + pythonScript: string; + requirementsContents: string; + testCommand: string; + testFilePath: string; + testFileContents: string; + pattern: string; + searchDirectory: string; + lang: string; + customScorerType: string; + scorerParams: string; +} + +const emptyScorerForm = (): ScorerFormData => ({ + name: "", + weight: "1.0", + type: "command_scorer", + command: "", + bashScript: "", + pythonScript: "", + requirementsContents: "", + testCommand: "", + testFilePath: "", + testFileContents: "", + pattern: "", + searchDirectory: "", + lang: "", + customScorerType: "", + scorerParams: "", +}); + +interface FormData { + name: string; + problemStatement: string; + referenceOutput: string; + environmentSource: EnvironmentSource; + environmentId: string; + workingDirectory: string; + scorers: ScorerFormData[]; + metadata: Record; + requiredEnvVars: string[]; + requiredSecrets: string[]; + validationType: ValidationType; + scorerTimeout: string; +} + +type FieldKey = + | "submit" + | "name" + | "problemStatement" + | "referenceOutput" + | "environmentSource" + | "environmentId" + | "workingDirectory" + | "scorers" + | "metadata" + | "requiredEnvVars" + | "requiredSecrets" + | "validationType" + | "scorerTimeout"; + +export function ScenarioCreateScreen() { + const { goBack } = useNavigation(); + const [screenState, setScreenState] = React.useState("form"); + const [activeFieldIndex, setActiveFieldIndex] = React.useState(0); + const [error, setError] = React.useState(null); + const [validationError, setValidationError] = React.useState( + null, + ); + const [editingScorerIndex, setEditingScorerIndex] = React.useState(-1); + const [scorerForm, setScorerForm] = + React.useState(emptyScorerForm()); + const [scorerFieldIndex, setScorerFieldIndex] = React.useState(0); + + const [formData, setFormData] = React.useState({ + name: "", + problemStatement: "", + referenceOutput: "", + environmentSource: "none", + environmentId: "", + workingDirectory: "", + scorers: [{ ...emptyScorerForm(), name: "default" }], + metadata: {}, + requiredEnvVars: [], + requiredSecrets: [], + validationType: "UNSPECIFIED", + scorerTimeout: "", + }); + + // Metadata state + const [inMetadataSection, setInMetadataSection] = React.useState(false); + const [metadataKey, setMetadataKey] = React.useState(""); + const [metadataValue, setMetadataValue] = React.useState(""); + const [metadataInputMode, setMetadataInputMode] = React.useState< + "key" | "value" | null + >(null); + const [selectedMetadataIndex, setSelectedMetadataIndex] = React.useState(0); + + // List manager expansion state + const [envVarsExpanded, setEnvVarsExpanded] = React.useState(false); + const [secretsExpanded, setSecretsExpanded] = React.useState(false); + + useExitOnCtrlC(); + + const visibleFields = React.useMemo((): Array<{ + key: FieldKey; + label: string; + type: "text" | "select" | "action" | "list" | "metadata" | "scorers"; + }> => { + const f: Array<{ + key: FieldKey; + label: string; + type: "text" | "select" | "action" | "list" | "metadata" | "scorers"; + }> = [ + { key: "submit", label: "Create Scenario", type: "action" }, + { key: "name", label: "Name (required)", type: "text" }, + { + key: "problemStatement", + label: "Problem Statement (required)", + type: "text", + }, + { key: "referenceOutput", label: "Reference Output", type: "text" }, + { key: "environmentSource", label: "Environment", type: "select" }, + ]; + if ( + formData.environmentSource === "blueprint" || + formData.environmentSource === "snapshot" + ) { + f.push({ + key: "environmentId", + label: + formData.environmentSource === "blueprint" + ? "Blueprint ID" + : "Snapshot ID", + type: "text", + }); + f.push({ + key: "workingDirectory", + label: "Working Directory", + type: "text", + }); + } + f.push({ key: "scorers", label: "Scoring Functions", type: "scorers" }); + f.push({ key: "metadata", label: "Metadata (optional)", type: "metadata" }); + f.push({ + key: "requiredEnvVars", + label: "Required Env Vars", + type: "list", + }); + f.push({ + key: "requiredSecrets", + label: "Required Secrets", + type: "list", + }); + f.push({ key: "validationType", label: "Validation Type", type: "select" }); + f.push({ + key: "scorerTimeout", + label: "Scorer Timeout (sec)", + type: "text", + }); + return f; + }, [formData.environmentSource]); + + const activeField = visibleFields[activeFieldIndex]?.key; + + const handleEnvSourceSelect = useFormSelectNavigation( + formData.environmentSource, + ENVIRONMENT_OPTIONS, + (v) => + setFormData((prev) => ({ + ...prev, + environmentSource: v as EnvironmentSource, + })), + activeField === "environmentSource", + ); + + const handleValidationSelect = useFormSelectNavigation( + formData.validationType, + VALIDATION_OPTIONS, + (v) => + setFormData((prev) => ({ + ...prev, + validationType: v as ValidationType, + })), + activeField === "validationType", + ); + + const handleScorerTypeSelect = useFormSelectNavigation( + scorerForm.type, + SCORER_TYPE_OPTIONS, + (v) => setScorerForm((prev) => ({ ...prev, type: v as ScorerType })), + screenState === "editing-scorer" && scorerFieldIndex === 2, + ); + + const buildScorerObject = ( + s: ScorerFormData, + ): ScenarioCreateParams["scoring_contract"]["scoring_function_parameters"][0] => { + const weight = parseFloat(s.weight) || 1.0; + let scorer: any; + + switch (s.type) { + case "command_scorer": + scorer = { type: "command_scorer" as const, command: s.command }; + break; + case "bash_script_scorer": + scorer = { + type: "bash_script_scorer" as const, + bash_script: s.bashScript, + }; + break; + case "python_script_scorer": + scorer = { + type: "python_script_scorer" as const, + python_script: s.pythonScript, + ...(s.requirementsContents + ? { requirements_contents: s.requirementsContents } + : {}), + }; + break; + case "test_based_scorer": + scorer = { + type: "test_based_scorer" as const, + test_command: s.testCommand, + ...(s.testFilePath + ? { + test_files: [ + { + file_path: s.testFilePath, + file_contents: s.testFileContents, + }, + ], + } + : {}), + }; + break; + case "ast_grep_scorer": + scorer = { + type: "ast_grep_scorer" as const, + pattern: s.pattern, + search_directory: s.searchDirectory, + ...(s.lang ? { lang: s.lang } : {}), + }; + break; + case "custom_scorer": { + let parsedParams: Record | undefined; + if (s.scorerParams) { + try { + parsedParams = JSON.parse(s.scorerParams); + } catch { + throw new Error(`Invalid JSON in scorer params for "${s.name}"`); + } + } + scorer = { + type: "custom_scorer" as const, + custom_scorer_type: s.customScorerType, + ...(parsedParams ? { scorer_params: parsedParams } : {}), + }; + break; + } + } + + return { name: s.name, weight, scorer }; + }; + + const handleSubmit = async () => { + if (!formData.name.trim()) { + setValidationError("Name is required"); + const idx = visibleFields.findIndex((f) => f.key === "name"); + if (idx >= 0) setActiveFieldIndex(idx); + return; + } + if (!formData.problemStatement.trim()) { + setValidationError("Problem statement is required"); + const idx = visibleFields.findIndex((f) => f.key === "problemStatement"); + if (idx >= 0) setActiveFieldIndex(idx); + return; + } + if (formData.scorers.length === 0) { + setValidationError("At least one scoring function is required"); + return; + } + + const totalWeight = formData.scorers.reduce( + (sum, s) => sum + (parseFloat(s.weight) || 0), + 0, + ); + if (Math.abs(totalWeight - 1.0) > 0.01) { + setValidationError( + `Scoring function weights must sum to 1.0 (currently ${totalWeight.toFixed(2)})`, + ); + return; + } + + setError(null); + setValidationError(null); + setScreenState("creating"); + + try { + const params: ScenarioCreateParams = { + name: formData.name.trim(), + input_context: { + problem_statement: formData.problemStatement.trim(), + }, + scoring_contract: { + scoring_function_parameters: formData.scorers.map(buildScorerObject), + }, + }; + + if (formData.referenceOutput.trim()) { + params.reference_output = formData.referenceOutput.trim(); + } + + if ( + formData.environmentSource !== "none" && + formData.environmentId.trim() + ) { + params.environment_parameters = { + ...(formData.environmentSource === "blueprint" + ? { blueprint_id: formData.environmentId.trim() } + : { snapshot_id: formData.environmentId.trim() }), + ...(formData.workingDirectory.trim() + ? { working_directory: formData.workingDirectory.trim() } + : {}), + }; + } + + if (Object.keys(formData.metadata).length > 0) { + params.metadata = formData.metadata; + } + + const envVars = formData.requiredEnvVars.filter((v) => v.trim()); + if (envVars.length > 0) { + params.required_environment_variables = envVars; + } + + const secrets = formData.requiredSecrets.filter((v) => v.trim()); + if (secrets.length > 0) { + params.required_secret_names = secrets; + } + + if (formData.scorerTimeout.trim()) { + const timeout = parseInt(formData.scorerTimeout, 10); + if (isNaN(timeout)) { + setValidationError("Scorer timeout must be a number"); + setScreenState("form"); + return; + } + params.scorer_timeout_sec = timeout; + } + + if (formData.validationType !== "UNSPECIFIED") { + params.validation_type = formData.validationType; + } + + await createScenario(params); + goBack(); + } catch (err) { + setError(err as Error); + setScreenState("error"); + } + }; + + // Scorer type-specific fields + const getScorerFields = ( + type: ScorerType, + ): Array<{ key: string; label: string; placeholder: string }> => { + switch (type) { + case "command_scorer": + return [{ key: "command", label: "Command", placeholder: "pytest -v" }]; + case "bash_script_scorer": + return [ + { + key: "bashScript", + label: "Bash Script", + placeholder: 'echo "score=1.0"', + }, + ]; + case "python_script_scorer": + return [ + { + key: "pythonScript", + label: "Python Script", + placeholder: 'print("score=1.0")', + }, + { + key: "requirementsContents", + label: "Requirements (optional)", + placeholder: "pytest>=7.0", + }, + ]; + case "test_based_scorer": + return [ + { + key: "testCommand", + label: "Test Command", + placeholder: "pytest tests/", + }, + { + key: "testFilePath", + label: "Test File Path (optional)", + placeholder: "tests/test_solution.py", + }, + { + key: "testFileContents", + label: "Test File Contents (optional)", + placeholder: "def test_solution(): ...", + }, + ]; + case "ast_grep_scorer": + return [ + { key: "pattern", label: "Pattern", placeholder: "$$$" }, + { + key: "searchDirectory", + label: "Search Directory", + placeholder: "src/", + }, + { key: "lang", label: "Language (optional)", placeholder: "python" }, + ]; + case "custom_scorer": + return [ + { + key: "customScorerType", + label: "Custom Scorer Type", + placeholder: "my_scorer", + }, + { + key: "scorerParams", + label: "Scorer Params (JSON, optional)", + placeholder: '{"key": "value"}', + }, + ]; + } + }; + + const scorerEditorFields = React.useMemo(() => { + const base = [ + { key: "name", label: "Scorer Name", placeholder: "default" }, + { key: "weight", label: "Weight (0-1)", placeholder: "1.0" }, + { key: "type", label: "Scorer Type", placeholder: "" }, + ]; + return [...base, ...getScorerFields(scorerForm.type)]; + }, [scorerForm.type]); + + // Scorer editor input + useInput( + (input, key) => { + if (handleScorerTypeSelect(input, key)) return; + + if (key.escape || input === "q") { + setScreenState("form"); + setEditingScorerIndex(-1); + return; + } + + const saveScorer = () => { + if (!scorerForm.name.trim()) return; + const newScorers = [...formData.scorers]; + if (editingScorerIndex >= 0 && editingScorerIndex < newScorers.length) { + newScorers[editingScorerIndex] = { ...scorerForm }; + } else { + newScorers.push({ ...scorerForm }); + } + setFormData((prev) => ({ ...prev, scorers: newScorers })); + setScreenState("form"); + setEditingScorerIndex(-1); + }; + + if (input === "s" && key.ctrl) { + saveScorer(); + return; + } + + if (key.return && scorerFieldIndex === scorerEditorFields.length) { + saveScorer(); + return; + } + + if (key.upArrow) { + setScorerFieldIndex((prev) => Math.max(0, prev - 1)); + return; + } + if (key.downArrow) { + setScorerFieldIndex((prev) => + Math.min(scorerEditorFields.length, prev + 1), + ); + return; + } + }, + { isActive: screenState === "editing-scorer" }, + ); + + // Main form input + useInput( + (input, key) => { + if (screenState === "error") { + if (input === "r" || key.return) { + setError(null); + setScreenState("form"); + } else if (input === "q" || key.escape) { + goBack(); + } + return; + } + + if (screenState !== "form") return; + + if (handleEnvSourceSelect(input, key)) return; + if (handleValidationSelect(input, key)) return; + + if (input === "q" || key.escape) { + goBack(); + return; + } + + if (input === "s" && key.ctrl) { + handleSubmit(); + return; + } + + if (activeField === "scorers") { + if (key.return) { + setScorerForm(emptyScorerForm()); + setEditingScorerIndex(formData.scorers.length); + setScorerFieldIndex(0); + setScreenState("editing-scorer"); + return; + } + const editMatch = input.match(/^e(\d+)$/); + if (editMatch) { + const idx = parseInt(editMatch[1], 10); + if (idx >= 0 && idx < formData.scorers.length) { + setScorerForm({ ...formData.scorers[idx] }); + setEditingScorerIndex(idx); + setScorerFieldIndex(0); + setScreenState("editing-scorer"); + } + return; + } + const deleteMatch = input.match(/^d(\d+)$/); + if (deleteMatch) { + const idx = parseInt(deleteMatch[1], 10); + if (idx >= 0 && idx < formData.scorers.length) { + setFormData((prev) => ({ + ...prev, + scorers: prev.scorers.filter((_, i) => i !== idx), + })); + } + return; + } + } + + if (activeField === "metadata" && key.return) { + setInMetadataSection(true); + setSelectedMetadataIndex(0); + return; + } + + if (activeField === "requiredEnvVars" && key.return) { + setEnvVarsExpanded(true); + return; + } + + if (activeField === "requiredSecrets" && key.return) { + setSecretsExpanded(true); + return; + } + + if (key.return) { + handleSubmit(); + return; + } + + if (key.upArrow || (key.tab && key.shift)) { + setActiveFieldIndex((prev) => Math.max(0, prev - 1)); + return; + } + + if (key.downArrow || (key.tab && !key.shift)) { + setActiveFieldIndex((prev) => + Math.min(visibleFields.length - 1, prev + 1), + ); + return; + } + }, + { + isActive: !inMetadataSection && !envVarsExpanded && !secretsExpanded, + }, + ); + + // Metadata section input + useInput( + (input, key) => { + const metadataKeys = Object.keys(formData.metadata); + const maxIndex = metadataKeys.length + 1; + + if (metadataInputMode) { + if (metadataInputMode === "key" && key.return && metadataKey.trim()) { + setMetadataInputMode("value"); + return; + } else if (metadataInputMode === "value" && key.return) { + if (metadataKey.trim() && metadataValue.trim()) { + setFormData((prev) => ({ + ...prev, + metadata: { + ...prev.metadata, + [metadataKey.trim()]: metadataValue.trim(), + }, + })); + } + setMetadataKey(""); + setMetadataValue(""); + setMetadataInputMode(null); + setSelectedMetadataIndex(0); + return; + } else if (key.escape) { + setMetadataKey(""); + setMetadataValue(""); + setMetadataInputMode(null); + return; + } else if (key.tab) { + setMetadataInputMode(metadataInputMode === "key" ? "value" : "key"); + return; + } + return; + } + + if (key.upArrow && selectedMetadataIndex > 0) { + setSelectedMetadataIndex(selectedMetadataIndex - 1); + } else if (key.downArrow && selectedMetadataIndex < maxIndex) { + setSelectedMetadataIndex(selectedMetadataIndex + 1); + } else if (key.return) { + if (selectedMetadataIndex === 0) { + setMetadataKey(""); + setMetadataValue(""); + setMetadataInputMode("key"); + } else if (selectedMetadataIndex === maxIndex) { + setInMetadataSection(false); + setSelectedMetadataIndex(0); + } else { + const keyToEdit = metadataKeys[selectedMetadataIndex - 1]; + setMetadataKey(keyToEdit || ""); + setMetadataValue(formData.metadata[keyToEdit] || ""); + const newMetadata = { ...formData.metadata }; + delete newMetadata[keyToEdit]; + setFormData((prev) => ({ ...prev, metadata: newMetadata })); + setMetadataInputMode("key"); + } + } else if ( + (input === "d" || key.delete) && + selectedMetadataIndex >= 1 && + selectedMetadataIndex <= metadataKeys.length + ) { + const keyToDelete = metadataKeys[selectedMetadataIndex - 1]; + const newMetadata = { ...formData.metadata }; + delete newMetadata[keyToDelete]; + setFormData((prev) => ({ ...prev, metadata: newMetadata })); + if (selectedMetadataIndex > Object.keys(newMetadata).length) { + setSelectedMetadataIndex( + Math.max(0, Object.keys(newMetadata).length), + ); + } + } else if (key.escape || input === "q") { + setInMetadataSection(false); + setSelectedMetadataIndex(0); + setMetadataKey(""); + setMetadataValue(""); + setMetadataInputMode(null); + } + }, + { isActive: inMetadataSection }, + ); + + const breadcrumbItems = [ + { label: "Home" }, + { label: "Benchmarks" }, + { label: "Create Scenario", active: true }, + ]; + + if (screenState === "creating") { + return ( + <> + + + + ); + } + + if (screenState === "error") { + return ( + <> + + + + + ); + } + + // Scorer editor screen + if (screenState === "editing-scorer") { + return ( + <> + + + + {scorerEditorFields.map((field, idx) => { + const isActive = scorerFieldIndex === idx; + + if (field.key === "type") { + return ( + + setScorerForm((prev) => ({ + ...prev, + type: v as ScorerType, + })) + } + isActive={isActive} + /> + ); + } + + const value = + (scorerForm as unknown as Record)[field.key] || + ""; + return ( + + setScorerForm((prev) => ({ ...prev, [field.key]: v })) + } + isActive={isActive} + placeholder={field.placeholder} + /> + ); + })} + + + + + + + ); + } + + // Render metadata section + const renderMetadata = (isActive: boolean) => { + if (!inMetadataSection) { + return ( + + + + {isActive ? figures.pointer : " "} Metadata (optional):{" "} + + + {Object.keys(formData.metadata).length > 0 + ? `${Object.keys(formData.metadata).length} item(s)` + : "None"} + + {isActive && ( + + {" "} + [Enter to manage] + + )} + + {Object.keys(formData.metadata).length > 0 && ( + + + + )} + + ); + } + + const metadataKeys = Object.keys(formData.metadata); + const maxIndex = metadataKeys.length + 1; + + return ( + + + {figures.hamburger} Manage Metadata + + {metadataInputMode && ( + + + {metadataInputMode === "key" ? ( + <> + Key: + + + ) : ( + Key: {metadataKey} + )} + + + {metadataInputMode === "value" ? ( + <> + Value: + + + ) : ( + Value: {metadataValue} + )} + + + )} + {!metadataInputMode && ( + <> + + + {selectedMetadataIndex === 0 ? figures.pointer : " "}{" "} + + + + Add new metadata + + + {metadataKeys.map((key, index) => { + const itemIndex = index + 1; + const isSelected = selectedMetadataIndex === itemIndex; + return ( + + + {isSelected ? figures.pointer : " "}{" "} + + + {key}: {formData.metadata[key]} + + + ); + })} + + + {selectedMetadataIndex === maxIndex + ? figures.pointer + : " "}{" "} + + + {figures.tick} Done + + + + )} + + + {metadataInputMode + ? `[Tab] Switch • [Enter] ${metadataInputMode === "key" ? "Next" : "Save"} • [esc] Cancel` + : `${figures.arrowUp}${figures.arrowDown} Navigate • [Enter] Select • [d] Delete • [esc] Back`} + + + + ); + }; + + // Render scorers section + const renderScorers = (isActive: boolean) => { + return ( + + + + {isActive ? figures.pointer : " "} Scoring Functions:{" "} + + {formData.scorers.length} scorer(s) + {isActive && ( + + {" "} + [Enter to add] + + )} + + {formData.scorers.map((s, idx) => ( + + + {figures.pointer} {s.name || "(unnamed)"} ({s.type}, weight:{" "} + {s.weight}) + + {isActive && ( + + {" "} + [e={idx} to edit, d={idx} to delete] + + )} + + ))} + + ); + }; + + // Main form + return ( + <> + + + + + {figures.info} Create Scenario — define a repeatable + AI coding evaluation with environment and scoring criteria. + + + + {validationError && ( + + + {figures.cross} {validationError} + + + )} + + + {visibleFields.map((field, idx) => { + const isActive = activeFieldIndex === idx; + + if (field.type === "action") { + return ( + + ); + } + + if (field.type === "text") { + let value = ""; + let placeholder = ""; + if (field.key === "name") { + value = formData.name; + placeholder = "my-scenario"; + } else if (field.key === "problemStatement") { + value = formData.problemStatement; + placeholder = "Fix the failing tests in ..."; + } else if (field.key === "referenceOutput") { + value = formData.referenceOutput; + placeholder = "(optional) expected output or diff"; + } else if (field.key === "environmentId") { + value = formData.environmentId; + placeholder = + formData.environmentSource === "blueprint" + ? "bpt_..." + : "snp_..."; + } else if (field.key === "workingDirectory") { + value = formData.workingDirectory; + placeholder = "/app"; + } else if (field.key === "scorerTimeout") { + value = formData.scorerTimeout; + placeholder = "1800"; + } + + return ( + { + setFormData((prev) => ({ + ...prev, + [field.key]: newValue, + })); + if (validationError) setValidationError(null); + }} + onSubmit={handleSubmit} + isActive={isActive} + placeholder={placeholder} + /> + ); + } + + if (field.type === "select") { + if (field.key === "environmentSource") { + return ( + + setFormData((prev) => ({ + ...prev, + environmentSource: v as EnvironmentSource, + })) + } + isActive={isActive} + /> + ); + } + if (field.key === "validationType") { + return ( + + setFormData((prev) => ({ + ...prev, + validationType: v as ValidationType, + })) + } + isActive={isActive} + /> + ); + } + } + + if (field.type === "scorers") { + return ( + + {renderScorers(isActive)} + + ); + } + + if (field.type === "metadata") { + return ( + + {renderMetadata(isActive)} + + ); + } + + if (field.type === "list") { + if (field.key === "requiredEnvVars") { + return ( + + setFormData((prev) => ({ ...prev, requiredEnvVars: items })) + } + isActive={isActive} + isExpanded={envVarsExpanded} + onExpandedChange={setEnvVarsExpanded} + itemPlaceholder="ENV_VAR_NAME" + addLabel="+ Add env var" + collapsedLabel="env var(s)" + /> + ); + } + if (field.key === "requiredSecrets") { + return ( + + setFormData((prev) => ({ ...prev, requiredSecrets: items })) + } + isActive={isActive} + isExpanded={secretsExpanded} + onExpandedChange={setSecretsExpanded} + itemPlaceholder="SECRET_NAME" + addLabel="+ Add secret" + collapsedLabel="secret(s)" + /> + ); + } + } + + return null; + })} + + + {!inMetadataSection && !envVarsExpanded && !secretsExpanded && ( + + )} + + ); +} diff --git a/src/services/benchmarkService.ts b/src/services/benchmarkService.ts index be9e807..9d70a95 100644 --- a/src/services/benchmarkService.ts +++ b/src/services/benchmarkService.ts @@ -26,6 +26,7 @@ export interface ListBenchmarksResult { export interface ListBenchmarkRunsOptions { limit: number; startingAfter?: string; + benchmarkId?: string; includeTotalCount?: boolean; } @@ -66,6 +67,9 @@ export async function listBenchmarkRuns( if (options.startingAfter) { queryParams.starting_after = options.startingAfter; } + if (options.benchmarkId) { + queryParams.benchmark_id = options.benchmarkId; + } const page = await client.benchmarkRuns.list(queryParams); const benchmarkRuns = page.runs || []; @@ -258,3 +262,13 @@ export async function createBenchmarkRun( // eslint-disable-next-line @typescript-eslint/no-explicit-any return (client.benchmarkRuns as any).create(createParams); } + +export async function cancelBenchmarkRun(id: string): Promise { + const client = getClient(); + return client.benchmarkRuns.cancel(id); +} + +export async function completeBenchmarkRun(id: string): Promise { + const client = getClient(); + return client.benchmarkRuns.complete(id); +} diff --git a/src/services/blueprintService.ts b/src/services/blueprintService.ts index 5f0ffb9..ff90537 100644 --- a/src/services/blueprintService.ts +++ b/src/services/blueprintService.ts @@ -4,6 +4,7 @@ import { getClient } from "../utils/client.js"; import type { Blueprint } from "../store/blueprintStore.js"; import type { + BlueprintCreateParams, BlueprintListParams, BlueprintView, } from "@runloop/api-client/resources/blueprints"; @@ -91,6 +92,74 @@ export async function listBlueprints( }; } +/** + * List public blueprints with pagination + */ +export async function listPublicBlueprints( + options: ListBlueprintsOptions, +): Promise { + const client = getClient(); + + const queryParams: { + limit?: number; + starting_after?: string; + name?: string; + include_total_count?: boolean; + } = { + limit: options.limit, + include_total_count: options.includeTotalCount === true, + }; + + if (options.startingAfter) { + queryParams.starting_after = options.startingAfter; + } + + if (options.search) { + queryParams.name = options.search; + } + + const page = (await client.blueprints.listPublic( + queryParams, + )) as unknown as BlueprintsCursorIDPage & { + total_count?: number; + }; + + const blueprints: Blueprint[] = []; + + if (page.blueprints && Array.isArray(page.blueprints)) { + page.blueprints.forEach((b: BlueprintView) => { + const MAX_ID_LENGTH = 100; + const MAX_NAME_LENGTH = 200; + const MAX_ARCH_LENGTH = 50; + const MAX_RESOURCES_LENGTH = 100; + + const architecture = b.parameters?.launch_parameters?.architecture; + const resources = b.parameters?.launch_parameters?.resource_size_request; + + blueprints.push({ + id: String(b.id || "").substring(0, MAX_ID_LENGTH), + name: String(b.name || "").substring(0, MAX_NAME_LENGTH), + status: b.status, + state: b.state, + create_time_ms: b.create_time_ms, + parameters: b.parameters, + architecture: architecture + ? String(architecture).substring(0, MAX_ARCH_LENGTH) + : undefined, + resources: resources + ? String(resources).substring(0, MAX_RESOURCES_LENGTH) + : undefined, + }); + }); + } + + return { + blueprints, + totalCount: page.total_count ?? blueprints.length, + hasMore: page.has_more || false, + }; +} + /** * Get a single blueprint by ID */ @@ -169,3 +238,65 @@ export async function getBlueprintLogs(id: string): Promise { return []; } + +export interface CreateBlueprintOptions { + name: string; + dockerfile?: string; + baseBlueprintId?: string; + systemSetupCommands?: string[]; + architecture?: string; + resourceSizeRequest?: string; + availablePorts?: number[]; + keepAliveTimeSeconds?: number; + metadata?: Record; +} + +export async function createBlueprint( + options: CreateBlueprintOptions, +): Promise { + const client = getClient(); + + const params: BlueprintCreateParams = { + name: options.name, + }; + + if (options.dockerfile) { + params.dockerfile = options.dockerfile; + } + if (options.baseBlueprintId) { + params.base_blueprint_id = options.baseBlueprintId; + } + if (options.systemSetupCommands && options.systemSetupCommands.length > 0) { + params.system_setup_commands = options.systemSetupCommands; + } + + const launchParameters: Record = {}; + if (options.architecture) { + launchParameters.architecture = options.architecture; + } + if (options.resourceSizeRequest) { + launchParameters.resource_size_request = options.resourceSizeRequest; + } + if (options.availablePorts && options.availablePorts.length > 0) { + launchParameters.available_ports = options.availablePorts; + } + if (options.keepAliveTimeSeconds) { + launchParameters.keep_alive_time_seconds = options.keepAliveTimeSeconds; + } + if (Object.keys(launchParameters).length > 0) { + params.launch_parameters = + launchParameters as BlueprintCreateParams["launch_parameters"]; + } + + if (options.metadata && Object.keys(options.metadata).length > 0) { + params.metadata = options.metadata; + } + + const blueprint = await client.blueprints.create(params); + const lp = blueprint.parameters?.launch_parameters; + return { + ...blueprint, + architecture: lp?.architecture ?? undefined, + resources: lp?.resource_size_request ?? undefined, + }; +} diff --git a/src/services/scenarioService.ts b/src/services/scenarioService.ts index 0b32209..ffe1529 100644 --- a/src/services/scenarioService.ts +++ b/src/services/scenarioService.ts @@ -3,6 +3,7 @@ */ import { getClient } from "../utils/client.js"; import type { + ScenarioCreateParams, ScenarioListParams, ScenarioView, } from "@runloop/api-client/resources/scenarios/scenarios"; @@ -59,3 +60,12 @@ export async function getScenario(id: string): Promise { const client = getClient(); return client.scenarios.retrieve(id); } + +export { type ScenarioCreateParams }; + +export async function createScenario( + params: ScenarioCreateParams, +): Promise { + const client = getClient(); + return client.scenarios.create(params); +} diff --git a/src/store/navigationStore.tsx b/src/store/navigationStore.tsx index 0b46513..e753d46 100644 --- a/src/store/navigationStore.tsx +++ b/src/store/navigationStore.tsx @@ -23,6 +23,7 @@ export type ScreenName = | "snapshot-detail" | "blueprint-list" | "blueprint-detail" + | "blueprint-create" | "blueprint-logs" | "network-policy-list" | "network-policy-detail" @@ -42,6 +43,8 @@ export type ScreenName = | "agent-create" | "axon-list" | "axon-detail" + | "axon-events" + | "axon-sql" | "object-list" | "object-detail" | "object-create" @@ -50,6 +53,7 @@ export type ScreenName = | "benchmark-detail" | "benchmark-run-list" | "benchmark-run-detail" + | "scenario-create" | "scenario-run-list" | "scenario-run-detail" | "benchmark-job-list" @@ -83,6 +87,7 @@ export interface RouteParams { executionId?: string; execCommand?: string; // Benchmark params + baseBlueprintId?: string; benchmarkId?: string; benchmarkRunId?: string; scenarioRunId?: string; diff --git a/src/utils/commands.ts b/src/utils/commands.ts index 19c9732..bbaf010 100644 --- a/src/utils/commands.ts +++ b/src/utils/commands.ts @@ -615,7 +615,7 @@ export function createProgram(): Command { object .command("list") .description("List objects") - .option("--limit ", "Max results", "20") + .option("-l, --limit ", "Max results", "20") .option("--starting-after ", "Starting point for pagination") .option("--name ", "Filter by name (partial match supported)") .option("--content-type ", "Filter by content type") @@ -722,7 +722,7 @@ export function createProgram(): Command { networkPolicy .command("list") .description("List network policies") - .option("--limit ", "Max results", "20") + .option("-l, --limit ", "Max results", "20") .option("--starting-after ", "Starting point for pagination") .option("--name ", "Filter by name") .option( @@ -808,7 +808,7 @@ export function createProgram(): Command { secret .command("list") .description("List all secrets") - .option("--limit ", "Max results", "20") + .option("-l, --limit ", "Max results", "20") .option( "-o, --output [format]", "Output format: text|json|yaml (default: json)", @@ -865,7 +865,7 @@ export function createProgram(): Command { .command("list") .description("List gateway configurations") .option("--name ", "Filter by name") - .option("--limit ", "Max results", "20") + .option("-l, --limit ", "Max results", "20") .option( "-o, --output [format]", "Output format: text|json|yaml (default: json)", @@ -955,7 +955,7 @@ export function createProgram(): Command { .command("list") .description("List MCP configurations") .option("--name ", "Filter by name") - .option("--limit ", "Max results", "20") + .option("-l, --limit ", "Max results", "20") .option( "-o, --output [format]", "Output format: text|json|yaml (default: json)", @@ -1069,7 +1069,7 @@ export function createProgram(): Command { axon .command("list") .description("List active axons") - .option("--limit ", "Max axons to return (0 = unlimited)", "0") + .option("-l, --limit ", "Max results", "20") .option( "--starting-after ", "Starting point for cursor pagination (axon ID)", @@ -1115,10 +1115,49 @@ export function createProgram(): Command { await scenarioInfo(id, options); }); + scenario + .command("create") + .description("Create a new custom scenario") + .requiredOption("--name ", "Scenario name") + .requiredOption("--problem-statement ", "Problem statement") + .option("--scoring-command ", "Simple command scorer (exit 0 = pass)") + .option( + "--scoring-file ", + "JSON file with full scoring_contract definition", + ) + .option("--blueprint ", "Blueprint ID for environment") + .option("--snapshot ", "Snapshot ID for environment") + .option("--working-directory ", "Working directory for scoring") + .option("--reference-output ", "Reference output text") + .option( + "--reference-output-file ", + "Path to file containing reference output", + ) + .option("--metadata ", "Metadata tags (format: key=value)") + .option( + "--required-env-vars ", + "Required environment variable names", + ) + .option("--required-secrets ", "Required secret names") + .option("--scorer-timeout ", "Scorer timeout in seconds") + .option( + "--validation-type ", + "Validation type: UNSPECIFIED|FORWARD|REVERSE|EVALUATION", + ) + .option( + "-o, --output [format]", + "Output format: text|json|yaml (default: json)", + ) + .action(async (options) => { + const { createScenarioCommand } = + await import("../commands/scenario/create.js"); + await createScenarioCommand(options); + }); + scenario .command("list") .description("List scenario runs") - .option("--limit ", "Max scenario runs to return (0 = unlimited)", "0") + .option("-l, --limit ", "Max results", "20") .option("--benchmark-run-id ", "Filter by benchmark run ID") .option( "-o, --output [format]", @@ -1130,6 +1169,38 @@ export function createProgram(): Command { await listScenarioRunsCommand(options); }); + // Benchmark run commands + const benchmarkRun = program + .command("benchmark-run") + .description("Manage benchmark runs") + .alias("bmr"); + + benchmarkRun + .command("cancel ") + .description("Cancel a running benchmark run") + .option( + "-o, --output [format]", + "Output format: text|json|yaml (default: json)", + ) + .action(async (id, options) => { + const { cancelBenchmarkRunCommand } = + await import("../commands/benchmark-run/cancel.js"); + await cancelBenchmarkRunCommand(id, options); + }); + + benchmarkRun + .command("complete ") + .description("Complete a benchmark run (finalize and score)") + .option( + "-o, --output [format]", + "Output format: text|json|yaml (default: json)", + ) + .action(async (id, options) => { + const { completeBenchmarkRunCommand } = + await import("../commands/benchmark-run/complete.js"); + await completeBenchmarkRunCommand(id, options); + }); + // Benchmark job commands const benchmarkJob = program .command("benchmark-job") @@ -1217,6 +1288,7 @@ export function createProgram(): Command { benchmarkJob .command("list") .description("List benchmark jobs") + .option("-l, --limit ", "Max results", "20") .option("--days ", "Show jobs from the last N days (default: 1)") .option("--all", "Show all jobs (no time filter)") .option( @@ -1247,7 +1319,7 @@ export function createProgram(): Command { .option("--search ", "Search by agent ID or name") .option("--public", "Show only public agents") .option("--private", "Show only private agents") - .option("--limit ", "Max results to return (0 = unlimited)", "0") + .option("-l, --limit ", "Max results", "20") .option("--starting-after ", "Cursor for pagination (agent ID)") .option( "-o, --output [format]",