Assertions

TraceAssert.NoErrors()

TraceAssert.no_errors()

TraceAssert.noErrors()

TraceAssert.MaxSteps(10)

TraceAssert.max_steps(10)

TraceAssert.maxSteps(10)

TraceAssert.MaxCost(0.10m)   // $0.10 max

TraceAssert.max_cost(0.10)   # $0.10 max

TraceAssert.maxCost(0.10)   // $0.10 max

TraceAssert.MaxDuration(5000)   // 5 seconds max

TraceAssert.max_duration(5000)   # 5 seconds max

TraceAssert.maxDuration(5000)   // 5 seconds max

TraceAssert.StepExists("classify")
TraceAssert.StepExists("answer")

TraceAssert.step_exists("classify")
TraceAssert.step_exists("answer")

TraceAssert.stepExists("classify")
TraceAssert.stepExists("answer")

TraceAssert.OutputContains("summarized")

TraceAssert.output_contains("summarized")

TraceAssert.outputContains("summarized")

TraceAssert.ToolCalled("web_search")

TraceAssert.tool_called("web_search")

TraceAssert.toolCalled("web_search")

public class OutputLengthAssertion : ITraceAssertion
{
    private readonly int _maxChars;
    public OutputLengthAssertion(int maxChars) => _maxChars = maxChars;

    public string? Evaluate(VevalExecutionContext ctx)
    {
        foreach (var step in ctx.Steps)
        {
            var len = step.Output?.ToString()?.Length ?? 0;
            if (len > _maxChars)
                return $"OutputLength: step '{step.Name}' output is {len} chars (max {_maxChars})";
        }
        return null;  // pass
    }
}

// Use it
TraceAssert.NoErrors(),
new OutputLengthAssertion(500),

from veval import ITraceAssertion

class OutputLengthAssertion(ITraceAssertion):
    def __init__(self, max_chars: int):
        self._max_chars = max_chars

    def evaluate(self, ctx) -> str | None:
        for step in ctx.steps:
            length = len(str(step.output)) if step.output is not None else 0
            if length > self._max_chars:
                return f"OutputLength: step '{step.name}' output is {length} chars (max {self._max_chars})"
        return None  # pass

# Use it
TraceAssert.no_errors(),
OutputLengthAssertion(500),

class OutputLengthAssertion {
  constructor(maxChars) {
    this.maxChars = maxChars;
  }

  evaluate(ctx) {
    for (const step of ctx.steps) {
      const len = step.output != null ? String(step.output).length : 0;
      if (len > this.maxChars)
        return `OutputLength: step '${step.name}' output is ${len} chars (max ${this.maxChars})`;
    }
    return null;  // pass
  }
}

// Use it
TraceAssert.noErrors(),
new OutputLengthAssertion(500),

await veval.RunScenarioAsync(
    scenarioName: "my-scenario",
    agent:        service.ExecuteAsync,
    // applied to every item
    scenarioAssertions: [
        TraceAssert.NoErrors(),
        TraceAssert.MaxSteps(10),
    ],
    items: [
        new ScenarioItem
        {
            Name  = "cheap question",
            Input = "What is 2+2?",
            // applied only to this item, on top of scenario assertions
            Assertions = [TraceAssert.MaxCost(0.001m)],
        },
    ]
);

await veval.run_scenario_async(
    scenario_name="my-scenario",
    agent=service.execute_async,
    # applied to every item
    scenario_assertions=[
        TraceAssert.no_errors(),
        TraceAssert.max_steps(10),
    ],
    items=[
        ScenarioItem(
            name="cheap question",
            input="What is 2+2?",
            # applied only to this item, on top of scenario assertions
            assertions=[TraceAssert.max_cost(0.001)],
        ),
    ],
)

await veval.runScenarioAsync(
  "my-scenario",
  (ctx) => service.execute(ctx),
  // applied to every item
  [
    TraceAssert.noErrors(),
    TraceAssert.maxSteps(10),
  ],
  [
    {
      name: "cheap question",
      input: "What is 2+2?",
      // applied only to this item, on top of scenario assertions
      assertions: [TraceAssert.maxCost(0.001)],
    },
  ]
);