commands.evals

Evals commands for Datalayer CLI.

evals_callback

@app.callback()
def evals_callback(ctx: typer.Context) -> None

Show evals command group help.

evals_ls

@app.command(name="ls")
def evals_ls(token: Optional[str] = typer.Option(None,
                                                 "--api-key",
                                                 help="API key."),
             billing_entity_uid: Optional[str] = typer.Option(
                 None,
                 "--billing-entity-uid",
                 help="Billing Entity UID context (organization/team/user).",
             ),
             run_environment: Optional[str] = typer.Option(
                 None,
                 "--run-environment",
                 help="Filter by run environment (ui/sdk)."),
             kind: Optional[str] = typer.Option(
                 None, "--kind", help="Filter by kind (batch/interactive)."),
             q: Optional[str] = typer.Option(None, "--q",
                                             help="Search query."),
             limit: int = typer.Option(50, "--limit", min=1, max=200),
             offset: int = typer.Option(0, "--offset", min=0),
             raw: bool = typer.Option(False,
                                      "--raw",
                                      help="Print raw JSON output.")) -> None

List all evalsets and their experiments.

evals_delete_top

@app.command(name="delete")
def evals_delete_top(
        evalset_id: str = typer.Argument(..., help="Evalset UID to delete."),
        yes: bool = typer.Option(False,
                                 "--yes",
                                 "-y",
                                 help="Skip the confirmation prompt."),
        token: Optional[str] = typer.Option(None, "--api-key",
                                            help="API key."),
        billing_entity_uid: Optional[str] = typer.Option(
            None,
            "--billing-entity-uid",
            help="Billing Entity UID context (organization/team/user).",
        )) -> None

Delete an evalset and its associated experiments, runs, and cases.

evals_list

@evals_app.command(name="ls")
def evals_list(token: Optional[str] = typer.Option(None,
                                                   "--api-key",
                                                   help="API key."),
               billing_entity_uid: Optional[str] = typer.Option(
                   None,
                   "--billing-entity-uid",
                   help="Billing Entity UID context (organization/team/user).",
               ),
               run_environment: Optional[str] = typer.Option(
                   None,
                   "--run-environment",
                   help="Filter by run environment (ui/sdk)."),
               kind: Optional[str] = typer.Option(
                   None, "--kind", help="Filter by kind (batch/interactive)."),
               q: Optional[str] = typer.Option(None,
                                               "--q",
                                               help="Search query."),
               limit: int = typer.Option(50, "--limit", min=1, max=200),
               offset: int = typer.Option(0, "--offset", min=0),
               raw: bool = typer.Option(
                   False, "--raw", help="Print raw JSON output.")) -> None

List evalsets.

evals_create

@evals_app.command(name="create")
def evals_create(
        name: Optional[str] = typer.Argument(None, help="Evalset name."),
        description: Optional[str] = typer.Option(None,
                                                  "--description",
                                                  help="Evalset description."),
        run_environment: Optional[str] = typer.Option(
            None,
            "--run-environment",
            help="Evalset run environment (ui/sdk)."),
        kind: Optional[str] = typer.Option(
            None, "--kind", help="Evalset kind (batch/interactive)."),
        spec_file: Optional[str] = typer.Option(
            None, "--spec-file", help="Path to evalset spec JSON file."),
        schema_json: Optional[str] = typer.Option(None,
                                                  "--schema-json",
                                                  help="Schema JSON object."),
        metadata_json: Optional[str] = typer.Option(
            None, "--metadata-json", help="Metadata JSON object."),
        cases_file: Optional[str] = typer.Option(
            None, "--cases-file", help="Path to JSON array of cases."),
        evalset_evaluator_json: list[str] = typer.
    Option(
        [],
        "--evalset-evaluator-json",
        help=
        "Repeatable JSON object applied as an evalset-level evaluator for the evalset.",
    ),
        report_evaluator_json: list[str] = typer.
    Option(
        [],
        "--report-evaluator-json",
        help=
        "Repeatable JSON object applied as a report-level evaluator for the evalset.",
    ),
        case_evaluator_json: list[str] = typer.
    Option(
        [],
        "--case-evaluator-json",
        help=
        "Repeatable JSON object applied as a case evaluator to every case in the payload.",
    ),
        tags: list[str] = typer.Option([], "--tag", help="Repeatable tag."),
        token: Optional[str] = typer.Option(None, "--api-key",
                                            help="API key."),
        billing_entity_uid: Optional[str] = typer.Option(
            None,
            "--billing-entity-uid",
            help="Billing Entity UID context (organization/team/user).",
        ),
        raw: bool = typer.Option(False, "--raw",
                                 help="Print raw JSON output.")) -> None

Create an evalset.

evals_delete

@evals_app.command(name="delete")
def evals_delete(
        evalset_id: str = typer.Argument(..., help="Evalset ID."),
        token: Optional[str] = typer.Option(None, "--api-key",
                                            help="API key."),
        billing_entity_uid: Optional[str] = typer.Option(
            None,
            "--billing-entity-uid",
            help="Billing Entity UID context (organization/team/user).",
        )) -> None

Delete an evalset (cascade delete runs/experiments).

evals_report

@app.command(name="report")
def evals_report(
        evalset_id: Optional[str] = typer.Argument(
            None,
            help="Evalset ID to report. Defaults to latest updated evalset."),
        run_limit: int = typer.Option(50,
                                      "--run-limit",
                                      min=2,
                                      max=200,
                                      help="Runs fetched per experiment."),
        token: Optional[str] = typer.Option(None, "--api-key",
                                            help="API key."),
        billing_entity_uid: Optional[str] = typer.Option(
            None,
            "--billing-entity-uid",
            help="Billing Entity UID context (organization/team/user).",
        ),
        output_file: Optional[str] = typer.Option(
            None, "--output", help="Write markdown report to file."),
        export: bool = typer.
    Option(
        False,
        "--export",
        help=
        "Export timestamped report files report-<timestamp>.md and report-<timestamp>.csv.",
    ),
        raw: bool = typer.Option(False,
                                 "--raw",
                                 help="Print raw JSON report output.")
) -> None

Generate an evalset report in markdown with comparison combinations and ASCII plots.

evals_evaluate

@app.command(name="evaluate")
def evals_evaluate(
        evalset_spec: str = typer.
    Argument(
        ...,
        help="Path to an evalset spec JSON file (with cases and evaluators)."),
        outputs_file: str = typer.
    Option(
        ...,
        "--outputs",
        help=
        'JSON file of agent outputs aligned with the evalset cases (list of strings or {text} objects, or {"outputs": [...]}).',
    ),
        statuses_file: Optional[str] = typer.
    Option(
        None,
        "--statuses",
        help="Optional JSON file of per-case run statuses aligned with cases.",
    ),
        output_file: Optional[str] = typer.Option(
            None,
            "--output",
            help="Write the computed metrics JSON to this file."),
        raw: bool = typer.Option(False,
                                 "--raw",
                                 help="Print the full metrics JSON.")) -> None

Run per-case and global evaluators over real agent outputs.

Grades the provided outputs against an evalset spec using the shared evals API (agent_runtimes.evals.remote.evaluate_evalset) and emits run metrics (case_results + evaluator_results). Callers produce outputs and delegate all evaluator execution here instead of re-implementing it.

experiments_list

@experiments_app.command(name="ls")
def experiments_list(
        evalset_id: Optional[str] = typer.Option(None,
                                                 "--evalset-id",
                                                 help="Filter by evalset ID."),
        status: Optional[str] = typer.Option(None,
                                             "--status",
                                             help="Filter by status."),
        limit: int = typer.Option(50, "--limit", min=1, max=200),
        offset: int = typer.Option(0, "--offset", min=0),
        token: Optional[str] = typer.Option(None, "--api-key",
                                            help="API key."),
        billing_entity_uid: Optional[str] = typer.Option(
            None,
            "--billing-entity-uid",
            help="Billing Entity UID context (organization/team/user).",
        ),
        raw: bool = typer.Option(False, "--raw",
                                 help="Print raw JSON output.")) -> None

List evalset experiments.

experiments_create

@experiments_app.command(name="create")
def experiments_create(
        name: Optional[str] = typer.Argument(None, help="Experiment name."),
        evalset_id: Optional[str] = typer.Option(None,
                                                 "--evalset-id",
                                                 help="Evalset ID."),
        description: Optional[str] = typer.Option(None,
                                                  "--description",
                                                  help="Description."),
        status: Optional[str] = typer.Option(None,
                                             "--status",
                                             help="Initial status."),
        spec_file: Optional[str] = typer.Option(
            None, "--spec-file", help="Path to experimentspec JSON file."),
        agent_spec_id: Optional[str] = typer.Option(
            None, "--agent-spec-id", help="Single agentspec id."),
        agent_spec_ids: Optional[str] = typer.
    Option(
        None,
        "--agent-spec-ids",
        help="Comma-separated agentspec ids for multi-experiment creation.",
    ),
        config_json: Optional[str] = typer.Option(None,
                                                  "--config-json",
                                                  help="Config JSON object."),
        summary_json: Optional[str] = typer.Option(
            None, "--summary-json", help="Summary JSON object."),
        tags: list[str] = typer.Option([], "--tag", help="Repeatable tag."),
        token: Optional[str] = typer.Option(None, "--api-key",
                                            help="API key."),
        billing_entity_uid: Optional[str] = typer.Option(
            None,
            "--billing-entity-uid",
            help="Billing Entity UID context (organization/team/user).",
        ),
        raw: bool = typer.Option(False, "--raw",
                                 help="Print raw JSON output.")) -> None

Create an evalset experiment.

runs_list

@runs_app.command(name="ls")
def runs_list(experiment_id: str = typer.Option(...,
                                                "--experiment-id",
                                                help="Experiment ID."),
              limit: int = typer.Option(50, "--limit", min=1, max=200),
              offset: int = typer.Option(0, "--offset", min=0),
              token: Optional[str] = typer.Option(None,
                                                  "--api-key",
                                                  help="API key."),
              billing_entity_uid: Optional[str] = typer.Option(
                  None,
                  "--billing-entity-uid",
                  help="Billing Entity UID context (organization/team/user).",
              ),
              raw: bool = typer.Option(False,
                                       "--raw",
                                       help="Print raw JSON output.")) -> None

List runs for an experiment.

runs_launch

@runs_app.command(name="launch")
def runs_launch(
        experiment_id: str = typer.Option(...,
                                          "--experiment-id",
                                          help="Experiment ID."),
        status: str = typer.Option("queued",
                                   "--status",
                                   help="Initial run status."),
        run_mode: Optional[str] = typer.Option(
            None, "--run-mode", help="Run mode hint (batch/interactive)."),
        agent_pod_name: Optional[str] = typer.Option(
            None,
            "--agent-pod-name",
            help="Agent pod for interactive execution."),
        submitted_code_file: Optional[str] = typer.Option(
            None,
            "--submitted-code-file",
            help="Python file to execute in interactive mode.",
        ),
        evalset_evaluator_json: list[str] = typer.
    Option(
        [],
        "--evalset-evaluator-json",
        help=
        "Repeatable JSON object for evalset-level evaluators attached to this run context.",
    ),
        report_evaluator_json: list[str] = typer.
    Option(
        [],
        "--report-evaluator-json",
        help=
        "Repeatable JSON object for evalset-level report evaluators attached to this run context.",
    ),
        metrics_json: Optional[str] = typer.Option(
            None, "--metrics-json", help="Inline metrics JSON object."),
        summary_json: Optional[str] = typer.Option(
            None, "--summary-json", help="Inline summary JSON object."),
        report_json: Optional[str] = typer.Option(
            None, "--report-json", help="Inline report JSON object."),
        metrics_file: Optional[str] = typer.Option(
            None, "--metrics-file", help="Path to metrics JSON object."),
        summary_file: Optional[str] = typer.Option(
            None, "--summary-file", help="Path to summary JSON object."),
        report_file: Optional[str] = typer.Option(
            None, "--report-file", help="Path to report JSON object."),
        started_at: Optional[str] = typer.Option(
            None, "--started-at", help="ISO timestamp override."),
        ended_at: Optional[str] = typer.Option(None,
                                               "--ended-at",
                                               help="ISO timestamp override."),
        token: Optional[str] = typer.Option(None, "--api-key",
                                            help="API key."),
        billing_entity_uid: Optional[str] = typer.Option(
            None,
            "--billing-entity-uid",
            help="Billing Entity UID context (organization/team/user).",
        )) -> None

Launch an evalset run on SaaS and tag it as CLI-launched.

runs_watch

@runs_app.command(name="watch")
def runs_watch(run_id: str = typer.Argument(..., help="Run ID."),
               interval_seconds: float = typer.Option(
                   3.0, "--interval", min=0.5, help="Polling interval."),
               timeout_seconds: int = typer.Option(600,
                                                   "--timeout",
                                                   min=5,
                                                   help="Timeout in seconds."),
               token: Optional[str] = typer.Option(None,
                                                   "--api-key",
                                                   help="API key."),
               billing_entity_uid: Optional[str] = typer.Option(
                   None,
                   "--billing-entity-uid",
                   help="Billing Entity UID context (organization/team/user).",
               )) -> None

Watch a run until completion/failure.

live_targets

@live_app.command(name="targets")
def live_targets(
        window: str = typer.Option("24h",
                                   "--window",
                                   help="Window: 1h, 6h, 24h, 7d, 30d."),
        limit: int = typer.Option(50, "--limit", min=1, max=200),
        token: Optional[str] = typer.Option(None, "--api-key",
                                            help="API key."),
        billing_entity_uid: Optional[str] = typer.Option(
            None,
            "--billing-entity-uid",
            help="Billing Entity UID context (organization/team/user).",
        ),
        raw: bool = typer.Option(False, "--raw",
                                 help="Print raw JSON output.")) -> None

List live monitoring targets.