googleapis/google/cloud/ces/v1beta/evaluation.proto

// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package google.cloud.ces.v1beta;

import "google/api/field_behavior.proto";
import "google/api/resource.proto";
import "google/cloud/ces/v1beta/app.proto";
import "google/cloud/ces/v1beta/common.proto";
import "google/cloud/ces/v1beta/example.proto";
import "google/cloud/ces/v1beta/fakes.proto";
import "google/cloud/ces/v1beta/golden_run.proto";
import "google/cloud/ces/v1beta/session_service.proto";
import "google/cloud/ces/v1beta/toolset_tool.proto";
import "google/protobuf/duration.proto";
import "google/protobuf/struct.proto";
import "google/protobuf/timestamp.proto";
import "google/rpc/status.proto";

option go_package = "cloud.google.com/go/ces/apiv1beta/cespb;cespb";
option java_multiple_files = true;
option java_outer_classname = "EvaluationProto";
option java_package = "com.google.cloud.ces.v1beta";

// Aggregated metrics for an evaluation or evaluation dataset.
message AggregatedMetrics {
  // Metrics for a single tool.
  message ToolMetrics {
    // Output only. The name of the tool.
    string tool = 1 [
      (google.api.field_behavior) = OUTPUT_ONLY,
      (google.api.resource_reference) = { type: "ces.googleapis.com/Tool" }
    ];

    // Output only. The number of times the tool passed.
    int32 pass_count = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The number of times the tool failed.
    int32 fail_count = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // Metrics for turn latency.
  message TurnLatencyMetrics {
    // Output only. The average latency of the turns.
    google.protobuf.Duration average_latency = 1
        [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // Metrics for tool call latency.
  message ToolCallLatencyMetrics {
    // Output only. The name of the tool.
    string tool = 1 [
      (google.api.field_behavior) = OUTPUT_ONLY,
      (google.api.resource_reference) = { type: "ces.googleapis.com/Tool" }
    ];

    // Output only. The average latency of the tool calls.
    google.protobuf.Duration average_latency = 2
        [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // Metrics for semantic similarity results.
  message SemanticSimilarityMetrics {
    // Output only. The average semantic similarity score (0-4).
    float score = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // Metrics for hallucination results.
  message HallucinationMetrics {
    // Output only. The average hallucination score (0 to 1).
    float score = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // Metrics aggregated per app version.
  message MetricsByAppVersion {
    // Output only. The app version ID.
    string app_version_id = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Metrics for each tool within this app version.
    repeated ToolMetrics tool_metrics = 2
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Metrics for semantic similarity within this app version.
    repeated SemanticSimilarityMetrics semantic_similarity_metrics = 3
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Metrics for hallucination within this app version.
    repeated HallucinationMetrics hallucination_metrics = 5
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Metrics for tool call latency within this app version.
    repeated ToolCallLatencyMetrics tool_call_latency_metrics = 6
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Metrics for turn latency within this app version.
    repeated TurnLatencyMetrics turn_latency_metrics = 7
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The number of times the evaluation passed.
    int32 pass_count = 8 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The number of times the evaluation failed.
    int32 fail_count = 9 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Metrics aggregated per turn within this app version.
    repeated MetricsByTurn metrics_by_turn = 4
        [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // Metrics aggregated per turn.
  message MetricsByTurn {
    // Output only. The turn index (0-based).
    int32 turn_index = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Metrics for each tool within this turn.
    repeated ToolMetrics tool_metrics = 2
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Metrics for semantic similarity within this turn.
    repeated SemanticSimilarityMetrics semantic_similarity_metrics = 3
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Metrics for hallucination within this turn.
    repeated HallucinationMetrics hallucination_metrics = 4
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Metrics for tool call latency within this turn.
    repeated ToolCallLatencyMetrics tool_call_latency_metrics = 5
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Metrics for turn latency within this turn.
    repeated TurnLatencyMetrics turn_latency_metrics = 6
        [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // Output only. Aggregated metrics, grouped by app version ID.
  repeated MetricsByAppVersion metrics_by_app_version = 3
      [(google.api.field_behavior) = OUTPUT_ONLY];
}

// An evaluation represents all of the information needed to simulate and
// evaluate an agent.
message Evaluation {
  option (google.api.resource) = {
    type: "ces.googleapis.com/Evaluation"
    pattern: "projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation}"
    plural: "evaluations"
    singular: "evaluation"
  };

  // Represents a single, checkable requirement.
  message GoldenExpectation {
    // The actual check to perform.
    oneof condition {
      // Optional. Check that a specific tool was called with the parameters.
      ToolCall tool_call = 2 [(google.api.field_behavior) = OPTIONAL];

      // Optional. Check that a specific tool had the expected response.
      ToolResponse tool_response = 3 [(google.api.field_behavior) = OPTIONAL];

      // Optional. Check that the agent responded with the correct response. The
      // role "agent" is implied.
      Message agent_response = 4 [(google.api.field_behavior) = OPTIONAL];

      // Optional. Check that the agent transferred the conversation to a
      // different agent.
      AgentTransfer agent_transfer = 5 [(google.api.field_behavior) = OPTIONAL];

      // Optional. Check that the agent updated the session variables to the
      // expected values. Used to also capture agent variable updates for golden
      // evals.
      google.protobuf.Struct updated_variables = 6
          [(google.api.field_behavior) = OPTIONAL];

      // Optional. The tool response to mock, with the parameters of interest
      // specified. Any parameters not specified will be hallucinated by the
      // LLM.
      ToolResponse mock_tool_response = 7
          [(google.api.field_behavior) = OPTIONAL];
    }

    // Optional. A note for this requirement, useful in reporting when specific
    // checks fail. E.g., "Check_Payment_Tool_Called".
    string note = 1 [(google.api.field_behavior) = OPTIONAL];
  }

  // A step defines a singular action to happen during the evaluation.
  message Step {
    // The step to perform.
    oneof step {
      // Optional. User input for the conversation.
      SessionInput user_input = 1 [(google.api.field_behavior) = OPTIONAL];

      // Optional. Transfer the conversation to a different agent.
      AgentTransfer agent_transfer = 2 [(google.api.field_behavior) = OPTIONAL];

      // Optional. Executes an expectation on the current turn.
      GoldenExpectation expectation = 3
          [(google.api.field_behavior) = OPTIONAL];
    }
  }

  // A golden turn defines a single turn in a golden conversation.
  message GoldenTurn {
    // Required. The steps required to replay a golden conversation.
    repeated Step steps = 1 [(google.api.field_behavior) = REQUIRED];

    // Optional. The root span of the golden turn for processing and maintaining
    // audio information.
    Span root_span = 2 [(google.api.field_behavior) = OPTIONAL];
  }

  // The steps required to replay a golden conversation.
  message Golden {
    // Required. The golden turns required to replay a golden conversation.
    repeated GoldenTurn turns = 2 [(google.api.field_behavior) = REQUIRED];

    // Optional. The evaluation expectations to evaluate the replayed
    // conversation against. Format:
    // `projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluationExpectation}`
    repeated string evaluation_expectations = 3 [
      (google.api.field_behavior) = OPTIONAL,
      (google.api.resource_reference) = {
        type: "ces.googleapis.com/EvaluationExpectation"
      }
    ];
  }

  // The expectation to evaluate the conversation produced by the simulation.
  message ScenarioExpectation {
    // The tool call and response pair to be evaluated.
    message ToolExpectation {
      // Required. The expected tool call, with the parameters of interest
      // specified. Any parameters not specified will be hallucinated by the
      // LLM.
      ToolCall expected_tool_call = 1 [(google.api.field_behavior) = REQUIRED];

      // Required. The tool response to mock, with the parameters of interest
      // specified. Any parameters not specified will be hallucinated by the
      // LLM.
      ToolResponse mock_tool_response = 2
          [(google.api.field_behavior) = REQUIRED];
    }

    // The expectation to evaluate the conversation produced by the simulation.
    oneof expectation {
      // Optional. The tool call and response pair to be evaluated.
      ToolExpectation tool_expectation = 1
          [(google.api.field_behavior) = OPTIONAL];

      // Optional. The agent response to be evaluated.
      Message agent_response = 2 [(google.api.field_behavior) = OPTIONAL];
    }
  }

  // The config for a scenario
  message Scenario {
    // Facts about the user as a key value pair.
    message UserFact {
      // Required. The name of the user fact.
      string name = 1 [(google.api.field_behavior) = REQUIRED];

      // Required. The value of the user fact.
      string value = 2 [(google.api.field_behavior) = REQUIRED];
    }

    // The expected behavior of the user task. This is used to determine whether
    // the scenario is successful.
    enum TaskCompletionBehavior {
      // Behavior unspecified. Will default to TASK_SATISFIED.
      TASK_COMPLETION_BEHAVIOR_UNSPECIFIED = 0;

      // The user task should be completed successfully.
      TASK_SATISFIED = 1;

      // The user task should be rejected.
      TASK_REJECTED = 2;
    }

    // The expected behavior of the user goal. This is used to determine whether
    // the scenario is successful.
    enum UserGoalBehavior {
      // Behavior unspecified. Will default to USER_GOAL_SATISFIED.
      USER_GOAL_BEHAVIOR_UNSPECIFIED = 0;

      // The user goal should be completed successfully.
      USER_GOAL_SATISFIED = 1;

      // The user goal should be rejected.
      USER_GOAL_REJECTED = 2;

      // Ignore the user goal status.
      USER_GOAL_IGNORED = 3;
    }

    // Required. The task to be targeted by the scenario.
    string task = 1 [(google.api.field_behavior) = REQUIRED];

    // Optional. The user facts to be used by the scenario.
    repeated UserFact user_facts = 4 [(google.api.field_behavior) = OPTIONAL];

    // Optional. The maximum number of turns to simulate. If not specified, the
    // simulation will continue until the task is complete.
    int32 max_turns = 5 [(google.api.field_behavior) = OPTIONAL];

    // Required. The rubrics to score the scenario against.
    repeated string rubrics = 2 [(google.api.field_behavior) = REQUIRED];

    // Required. The ScenarioExpectations to evaluate the conversation produced
    // by the user simulation.
    repeated ScenarioExpectation scenario_expectations = 3
        [(google.api.field_behavior) = REQUIRED];

    // Optional. Variables / Session Parameters as context for the session,
    // keyed by variable names. Members of this struct will override any default
    // values set by the system.
    //
    // Note, these are different from user facts, which are facts known to the
    // user. Variables are parameters known to the agent: i.e. MDN (phone
    // number) passed by the telephony system.
    google.protobuf.Struct variable_overrides = 6
        [(google.api.field_behavior) = OPTIONAL];

    // Optional. Deprecated. Use user_goal_behavior instead.
    TaskCompletionBehavior task_completion_behavior = 7
        [deprecated = true, (google.api.field_behavior) = OPTIONAL];

    // Optional. The expected behavior of the user goal.
    UserGoalBehavior user_goal_behavior = 8
        [(google.api.field_behavior) = OPTIONAL];

    // Optional. The evaluation expectations to evaluate the conversation
    // produced by the simulation against. Format:
    // `projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluationExpectation}`
    repeated string evaluation_expectations = 10 [
      (google.api.field_behavior) = OPTIONAL,
      (google.api.resource_reference) = {
        type: "ces.googleapis.com/EvaluationExpectation"
      }
    ];
  }

  // The inputs for the evaluation
  oneof inputs {
    // Optional. The golden steps to be evaluated.
    Golden golden = 11 [(google.api.field_behavior) = OPTIONAL];

    // Optional. The config for a scenario.
    Scenario scenario = 12 [(google.api.field_behavior) = OPTIONAL];
  }

  // Identifier. The unique identifier of this evaluation.
  // Format:
  // `projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation}`
  string name = 1 [(google.api.field_behavior) = IDENTIFIER];

  // Required. User-defined display name of the evaluation. Unique within an
  // App.
  string display_name = 2 [(google.api.field_behavior) = REQUIRED];

  // Optional. User-defined description of the evaluation.
  string description = 3 [(google.api.field_behavior) = OPTIONAL];

  // Optional. User defined tags to categorize the evaluation.
  repeated string tags = 4 [(google.api.field_behavior) = OPTIONAL];

  // Output only. List of evaluation datasets the evaluation belongs to.
  // Format:
  // `projects/{project}/locations/{location}/apps/{app}/evaluationDatasets/{evaluationDataset}`
  repeated string evaluation_datasets = 5 [
    (google.api.field_behavior) = OUTPUT_ONLY,
    (google.api.resource_reference) = {
      type: "ces.googleapis.com/EvaluationDataset"
    }
  ];

  // Output only. Timestamp when the evaluation was created.
  google.protobuf.Timestamp create_time = 8
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The user who created the evaluation.
  string created_by = 13 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Timestamp when the evaluation was last updated.
  google.protobuf.Timestamp update_time = 9
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The user who last updated the evaluation.
  string last_updated_by = 14 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The EvaluationRuns that this Evaluation is associated with.
  repeated string evaluation_runs = 15 [
    (google.api.field_behavior) = OUTPUT_ONLY,
    (google.api.resource_reference) = {
      type: "ces.googleapis.com/EvaluationRun"
    }
  ];

  // Output only. Etag used to ensure the object hasn't changed during a
  // read-modify-write operation. If the etag is empty, the update will
  // overwrite any concurrent changes.
  string etag = 10 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The aggregated metrics for this evaluation across all runs.
  AggregatedMetrics aggregated_metrics = 16
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The latest evaluation result for this evaluation.
  EvaluationResult last_completed_result = 17
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Whether the evaluation is invalid. This can happen if an
  // evaluation is referencing a tool, toolset, or agent that has since been
  // deleted.
  bool invalid = 18 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The last 10 evaluation results for this evaluation. This is
  // only populated if include_last_ten_results is set to true in the
  // ListEvaluationsRequest or GetEvaluationRequest.
  repeated EvaluationResult last_ten_results = 19
      [(google.api.field_behavior) = OUTPUT_ONLY];
}

// An evaluation dataset represents a set of evaluations that are grouped
// together basaed on shared tags.
message EvaluationDataset {
  option (google.api.resource) = {
    type: "ces.googleapis.com/EvaluationDataset"
    pattern: "projects/{project}/locations/{location}/apps/{app}/evaluationDatasets/{evaluation_dataset}"
    plural: "evaluationDatasets"
    singular: "evaluationDataset"
  };

  // Identifier. The unique identifier of this evaluation dataset.
  // Format:
  // `projects/{project}/locations/{location}/apps/{app}/evaluationDatasets/{evaluationDataset}`
  string name = 1 [(google.api.field_behavior) = IDENTIFIER];

  // Required. User-defined display name of the evaluation dataset. Unique
  // within an App.
  string display_name = 2 [(google.api.field_behavior) = REQUIRED];

  // Optional. Evaluations that are included in this dataset.
  repeated string evaluations = 3 [
    (google.api.field_behavior) = OPTIONAL,
    (google.api.resource_reference) = { type: "ces.googleapis.com/Evaluation" }
  ];

  // Output only. Timestamp when the evaluation dataset was created.
  google.protobuf.Timestamp create_time = 4
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Timestamp when the evaluation dataset was last updated.
  google.protobuf.Timestamp update_time = 5
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Etag used to ensure the object hasn't changed during a
  // read-modify-write operation. If the etag is empty, the update will
  // overwrite any concurrent changes.
  string etag = 6 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The user who created the evaluation dataset.
  string created_by = 7 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The user who last updated the evaluation dataset.
  string last_updated_by = 8 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The aggregated metrics for this evaluation dataset across all
  // runs.
  AggregatedMetrics aggregated_metrics = 9
      [(google.api.field_behavior) = OUTPUT_ONLY];
}

// An evaluation result represents the output of running an Evaluation.
message EvaluationResult {
  option (google.api.resource) = {
    type: "ces.googleapis.com/EvaluationResult"
    pattern: "projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation}/results/{evaluation_result}"
    plural: "evaluationResults"
    singular: "evaluationResult"
  };

  // Specifies the expectation and the result of that expectation.
  message GoldenExpectationOutcome {
    // The result of the tool invocation check.
    message ToolInvocationResult {
      // Output only. The tool invocation parameter correctness score. This
      // indicates the percent of parameters from the expected tool call that
      // were also present in the actual tool call.
      optional float parameter_correctness_score = 1
          [(google.api.field_behavior) = OUTPUT_ONLY];

      // Output only. The outcome of the tool invocation check. This is
      // determined by comparing the parameter_correctness_score to the
      // threshold. If the score is equal to or above the threshold, the outcome
      // will be PASS. Otherwise, the outcome will be FAIL.
      Outcome outcome = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

      // Output only. A free text explanation for the tool invocation result.
      string explanation = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
    }

    // The result of the expectation.
    oneof result {
      // Output only. The result of the tool call expectation.
      ToolCall observed_tool_call = 2
          [(google.api.field_behavior) = OUTPUT_ONLY];

      // Output only. The result of the tool response expectation.
      ToolResponse observed_tool_response = 3
          [(google.api.field_behavior) = OUTPUT_ONLY];

      // Output only. The result of the agent response expectation.
      Message observed_agent_response = 4
          [(google.api.field_behavior) = OUTPUT_ONLY];

      // Output only. The result of the agent transfer expectation.
      AgentTransfer observed_agent_transfer = 5
          [(google.api.field_behavior) = OUTPUT_ONLY];
    }

    // Output only. The expectation that was evaluated.
    Evaluation.GoldenExpectation expectation = 1
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The outcome of the expectation.
    Outcome outcome = 6 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The result of the semantic similarity check.
    SemanticSimilarityResult semantic_similarity_result = 7
        [deprecated = true, (google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The result of the tool invocation check.
    ToolInvocationResult tool_invocation_result = 8
        [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // The result of a single evaluation expectation.
  message EvaluationExpectationResult {
    // Output only. The evaluation expectation.
    // Format:
    // `projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluation_expectation}`
    string evaluation_expectation = 1 [
      (google.api.field_behavior) = OUTPUT_ONLY,
      (google.api.resource_reference) = {
        type: "ces.googleapis.com/EvaluationExpectation"
      }
    ];

    // Output only. The prompt that was used for the evaluation.
    string prompt = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The outcome of the evaluation expectation.
    Outcome outcome = 3 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The explanation for the result.
    string explanation = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // The result of a golden evaluation.
  message GoldenResult {
    // The result of running a single turn of the golden conversation.
    message TurnReplayResult {
      // Output only. The conversation that was generated for this turn.
      string conversation = 1 [
        (google.api.field_behavior) = OUTPUT_ONLY,
        (google.api.resource_reference) = {
          type: "ces.googleapis.com/Conversation"
        }
      ];

      // Output only. The outcome of each expectation.
      repeated GoldenExpectationOutcome expectation_outcome = 2
          [(google.api.field_behavior) = OUTPUT_ONLY];

      // Output only. The result of the hallucination check.
      HallucinationResult hallucination_result = 3
          [(google.api.field_behavior) = OUTPUT_ONLY];

      // Output only. Deprecated. Use OverallToolInvocationResult instead.
      float tool_invocation_score = 4
          [deprecated = true, (google.api.field_behavior) = OUTPUT_ONLY];

      // Output only. The overall tool ordered invocation score for this turn.
      // This indicates the overall percent of tools from the expected turn that
      // were actually invoked in the expected order.
      optional float tool_ordered_invocation_score = 5
          [(google.api.field_behavior) = OUTPUT_ONLY];

      // Output only. Duration of the turn.
      google.protobuf.Duration turn_latency = 6
          [(google.api.field_behavior) = OUTPUT_ONLY];

      // Output only. The latency of each tool call in the turn.
      repeated ToolCallLatency tool_call_latencies = 7
          [(google.api.field_behavior) = OUTPUT_ONLY];

      // Output only. The result of the semantic similarity check.
      SemanticSimilarityResult semantic_similarity_result = 8
          [(google.api.field_behavior) = OUTPUT_ONLY];

      // Output only. The result of the overall tool invocation check.
      OverallToolInvocationResult overall_tool_invocation_result = 9
          [(google.api.field_behavior) = OUTPUT_ONLY];

      // Output only. Information about the error that occurred during this
      // turn.
      EvaluationErrorInfo error_info = 10
          [(google.api.field_behavior) = OUTPUT_ONLY];

      // Output only. The latency of spans in the turn.
      repeated SpanLatency span_latencies = 11
          [(google.api.field_behavior) = OUTPUT_ONLY];
    }

    // Output only. The result of running each turn of the golden conversation.
    repeated TurnReplayResult turn_replay_results = 1
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The results of the evaluation expectations.
    repeated EvaluationExpectationResult evaluation_expectation_results = 2
        [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // The outcome of the evaluation against the rubric.
  message ScenarioRubricOutcome {
    // Output only. The rubric that was used to evaluate the conversation.
    string rubric = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The score of the conversation against the rubric.
    optional float score = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The rater's response to the rubric.
    string score_explanation = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // The outcome of a scenario expectation.
  message ScenarioExpectationOutcome {
    // The observed tool call and response.
    message ObservedToolCall {
      // Output only. The observed tool call.
      ToolCall tool_call = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

      // Output only. The observed tool response.
      ToolResponse tool_response = 2
          [(google.api.field_behavior) = OUTPUT_ONLY];
    }

    // The result of the expectation.
    oneof result {
      // Output only. The observed tool call.
      ObservedToolCall observed_tool_call = 2
          [(google.api.field_behavior) = OUTPUT_ONLY];

      // Output only. The observed agent response.
      Message observed_agent_response = 3
          [(google.api.field_behavior) = OUTPUT_ONLY];
    }

    // Output only. The expectation that was evaluated.
    Evaluation.ScenarioExpectation expectation = 1
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The outcome of the ScenarioExpectation.
    Outcome outcome = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // The outcome of a scenario evaluation.
  message ScenarioResult {
    // Output only. The conversation that was generated in the scenario.
    string conversation = 1 [
      (google.api.field_behavior) = OUTPUT_ONLY,
      (google.api.resource_reference) = {
        type: "ces.googleapis.com/Conversation"
      }
    ];

    // Output only. The task that was used when running the scenario for this
    // result.
    string task = 10 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The user facts that were used by the scenario for this
    // result.
    repeated Evaluation.Scenario.UserFact user_facts = 11
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The outcome of each expectation.
    repeated ScenarioExpectationOutcome expectation_outcomes = 2
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The outcome of the rubric.
    repeated ScenarioRubricOutcome rubric_outcomes = 3
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The result of the hallucination check.
    // There will be one hallucination result for each turn in the conversation.
    repeated HallucinationResult hallucination_result = 4
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The result of the task completion check.
    TaskCompletionResult task_completion_result = 5
        [deprecated = true, (google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The latency of each tool call execution in the conversation.
    repeated ToolCallLatency tool_call_latencies = 6
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The result of the user goal satisfaction check.
    UserGoalSatisfactionResult user_goal_satisfaction_result = 7
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Whether all expectations were satisfied for this turn.
    optional bool all_expectations_satisfied = 8
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Whether the task was completed for this turn. This is a
    // composite of all expectations satisfied, no hallucinations, and user goal
    // satisfaction.
    optional bool task_completed = 9
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The latency of spans in the conversation.
    repeated SpanLatency span_latencies = 12
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The results of the evaluation expectations.
    repeated EvaluationExpectationResult evaluation_expectation_results = 13
        [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // The latency of a span execution.
  message SpanLatency {
    // The type of span.
    // Additional values may be added in the future.
    enum Type {
      // Default value. This value is unused.
      TYPE_UNSPECIFIED = 0;

      // Tool call span.
      TOOL = 1;

      // User callback span.
      USER_CALLBACK = 2;

      // Guardrail span.
      GUARDRAIL = 3;

      // LLM span.
      LLM = 4;
    }

    // The identifier of the specific item based on its type.
    oneof identifier {
      // Output only. The resource name of the guardrail or tool spans.
      string resource = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

      // Output only. The toolset tool identifier.
      ToolsetTool toolset = 3 [(google.api.field_behavior) = OUTPUT_ONLY];

      // Output only. The name of the LLM span.
      string model = 4 [(google.api.field_behavior) = OUTPUT_ONLY];

      // Output only. The name of the user callback span.
      string callback = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
    }

    // Output only. The type of span.
    Type type = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The display name of the span. Applicable to tool and
    // guardrail spans.
    string display_name = 9 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The start time of span.
    google.protobuf.Timestamp start_time = 6
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The end time of span.
    google.protobuf.Timestamp end_time = 7
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The latency of span.
    google.protobuf.Duration execution_latency = 8
        [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // The latency of a tool call execution.
  message ToolCallLatency {
    // Output only. The name of the tool that got executed.
    // Format:
    // `projects/{project}/locations/{location}/apps/{app}/tools/{tool}`.
    string tool = 1 [
      (google.api.field_behavior) = OUTPUT_ONLY,
      (google.api.resource_reference) = { type: "ces.googleapis.com/Tool" }
    ];

    // Output only. The display name of the tool.
    string display_name = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The start time of the tool call execution.
    google.protobuf.Timestamp start_time = 3
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The end time of the tool call execution.
    google.protobuf.Timestamp end_time = 4
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The latency of the tool call execution.
    google.protobuf.Duration execution_latency = 5
        [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // The result of the hallucination check for a single turn.
  message HallucinationResult {
    // Output only. The hallucination score. Can be -1, 0, 1.
    optional int32 score = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The label associated with each score.
    // Score 1: Justified
    // Score 0: Not Justified
    // Score -1: No Claim To Assess
    string label = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The explanation for the hallucination score.
    string explanation = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // The result of a user goal satisfaction check for a conversation.
  message UserGoalSatisfactionResult {
    // Output only. The user task satisfaction score. Can be -1, 0, 1.
    optional int32 score = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The label associated with each score.
    // Score 1: User Task Satisfied
    // Score 0: User Task Not Satisfied
    // Score -1: User Task Unspecified
    string label = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The explanation for the user task satisfaction score.
    string explanation = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // The result of the semantic similarity check.
  message SemanticSimilarityResult {
    // Output only. The semantic similarity score. Can be 0, 1, 2, 3, or 4.
    optional int32 score = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The label associated with each score.
    // Score 4: Fully Consistent
    // Score 3: Mostly Consistent
    // Score 2: Partially Consistent (Minor Omissions)
    // Score 1: Largely Inconsistent (Major Omissions)
    // Score 0: Completely Inconsistent / Contradictory
    string label = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The explanation for the semantic similarity score.
    string explanation = 3 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The outcome of the semantic similarity check. This is
    // determined by comparing the score to the
    // semantic_similarity_success_threshold. If the score is equal to or above
    // the threshold, the outcome will be PASS. Otherwise, the outcome will be
    // FAIL.
    Outcome outcome = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // The result of the overall tool invocation check.
  message OverallToolInvocationResult {
    // The overall tool invocation score for this turn. This indicates the
    // overall percent of tools from the expected turn that were actually
    // invoked.
    optional float tool_invocation_score = 1;

    // Output only. The outcome of the tool invocation check. This is determined
    // by comparing the tool_invocation_score to the
    // overall_tool_invocation_correctness_threshold. If the
    // score is equal to or above the threshold, the outcome will be PASS.
    // Otherwise, the outcome will be FAIL.
    Outcome outcome = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // The result of the task completion check for the conversation.
  message TaskCompletionResult {
    // Output only. The task completion score. Can be -1, 0, 1
    optional int32 score = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The label associated with each score.
    // Score 1: Task Completed
    // Score 0: Task Not Completed
    // Score -1: User Goal Undefined
    string label = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The explanation for the task completion score.
    string explanation = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // The outcome of the evaluation or expectation.
  enum Outcome {
    // Evaluation outcome is not specified.
    OUTCOME_UNSPECIFIED = 0;

    // Evaluation/Expectation passed. In the case of an evaluation, this means
    // that all expectations were met.
    PASS = 1;

    // Evaluation/Expectation failed. In the case of an evaluation, this means
    // that at least one expectation was not met.
    FAIL = 2;
  }

  // The state of the evaluation result execution.
  enum ExecutionState {
    // Evaluation result execution state is not specified.
    EXECUTION_STATE_UNSPECIFIED = 0;

    // Evaluation result execution is running.
    RUNNING = 1;

    // Evaluation result execution has completed.
    COMPLETED = 2;

    // Evaluation result execution failed due to an internal error.
    ERROR = 3;
  }

  // The result of the evaluation. Only populated when the execution_state is
  // COMPLETED.
  oneof result {
    // Output only. The outcome of a golden evaluation.
    GoldenResult golden_result = 7 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The outcome of a scenario evaluation.
    ScenarioResult scenario_result = 8
        [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // Identifier. The unique identifier of the evaluation result.
  // Format:
  // `projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation}/results/{result}`
  string name = 1 [(google.api.field_behavior) = IDENTIFIER];

  // Required. Display name of the Evaluation Result. Unique within an
  // Evaluation. By default, it has the following format:
  // "<evaluation-display-name> result
  // - <timestamp>".
  string display_name = 2 [(google.api.field_behavior) = REQUIRED];

  // Output only. Timestamp when the evaluation result was created.
  google.protobuf.Timestamp create_time = 5
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The outcome of the evaluation. Only populated if
  // execution_state is COMPLETE.
  Outcome evaluation_status = 4 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The evaluation run that produced this result.
  // Format:
  // `projects/{project}/locations/{location}/apps/{app}/evaluationRuns/{evaluationRun}`
  string evaluation_run = 6 [
    (google.api.field_behavior) = OUTPUT_ONLY,
    (google.api.resource_reference) = {
      type: "ces.googleapis.com/EvaluationRun"
    }
  ];

  // Output only. The persona used to generate the conversation for the
  // evaluation result.
  EvaluationPersona persona = 17 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Error information for the evaluation result.
  EvaluationErrorInfo error_info = 15
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Deprecated: Use `error_info` instead. Errors encountered
  // during execution.
  google.rpc.Status error = 14
      [deprecated = true, (google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The user who initiated the evaluation run that resulted in
  // this result.
  string initiated_by = 9 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The app version used to generate the conversation that
  // resulted in this result. Format:
  // `projects/{project}/locations/{location}/apps/{app}/versions/{version}`
  string app_version = 10 [
    (google.api.field_behavior) = OUTPUT_ONLY,
    (google.api.resource_reference) = { type: "ces.googleapis.com/AppVersion" }
  ];

  // Output only. The display name of the `app_version` that the evaluation ran
  // against.
  string app_version_display_name = 13
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The changelog of the app version that the evaluation ran
  // against. This is populated if user runs evaluation on latest/draft.
  string changelog = 21 [
    (google.api.field_behavior) = OUTPUT_ONLY,
    (google.api.resource_reference) = { type: "ces.googleapis.com/Changelog" }
  ];

  // Output only. The create time of the changelog of the app version that the
  // evaluation ran against. This is populated if user runs evaluation on
  // latest/draft.
  google.protobuf.Timestamp changelog_create_time = 22
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The state of the evaluation result execution.
  ExecutionState execution_state = 11
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The evaluation thresholds for the result.
  EvaluationMetricsThresholds evaluation_metrics_thresholds = 12
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The configuration used in the evaluation run that resulted in
  // this result.
  EvaluationConfig config = 18 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The method used to run the golden evaluation.
  GoldenRunMethod golden_run_method = 19
      [(google.api.field_behavior) = OUTPUT_ONLY];
}

// An evaluation run represents an all the evaluation results from an
// evaluation execution.
message EvaluationRun {
  option (google.api.resource) = {
    type: "ces.googleapis.com/EvaluationRun"
    pattern: "projects/{project}/locations/{location}/apps/{app}/evaluationRuns/{evaluation_run}"
    plural: "evaluationRuns"
    singular: "evaluationRun"
  };

  // The progress of the evaluation run.
  message Progress {
    // Output only. Total number of evaluation results in this run.
    int32 total_count = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Number of completed evaluation results with an outcome of
    // FAIL. (EvaluationResult.execution_state is COMPLETED and
    // EvaluationResult.evaluation_status is FAIL).
    int32 failed_count = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Number of evaluation results that failed to execute.
    // (EvaluationResult.execution_state is ERROR).
    int32 error_count = 3 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Number of evaluation results that finished successfully.
    // (EvaluationResult.execution_state is COMPLETED).
    int32 completed_count = 4 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Number of completed evaluation results with an outcome of
    // PASS. (EvaluationResult.execution_state is COMPLETED and
    // EvaluationResult.evaluation_status is PASS).
    int32 passed_count = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // Contains the summary of passed and failed result counts for a specific
  // evaluation in an evaluation run.
  message EvaluationRunSummary {
    // Output only. Number of passed results for the associated Evaluation in
    // this run.
    int32 passed_count = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Number of failed results for the associated Evaluation in
    // this run.
    int32 failed_count = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Number of error results for the associated Evaluation in
    // this run.
    int32 error_count = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // The type of the evaluations in this run.
  // Additional values may be added in the future.
  enum EvaluationType {
    // Evaluation type is not specified.
    EVALUATION_TYPE_UNSPECIFIED = 0;

    // Golden evaluation.
    GOLDEN = 1;

    // Scenario evaluation.
    SCENARIO = 2;

    // Indicates the run includes a mix of golden and scenario evaluations.
    MIXED = 3;
  }

  // The state of the evaluation run.
  enum EvaluationRunState {
    // Evaluation run state is not specified.
    EVALUATION_RUN_STATE_UNSPECIFIED = 0;

    // Evaluation run is running.
    RUNNING = 1;

    // Evaluation run has completed.
    COMPLETED = 2;

    // The evaluation run has an error.
    ERROR = 3;
  }

  // Identifier. The unique identifier of the evaluation run.
  // Format:
  // `projects/{project}/locations/{location}/apps/{app}/evaluationRuns/{evaluationRun}`
  string name = 1 [(google.api.field_behavior) = IDENTIFIER];

  // Optional. User-defined display name of the evaluation run.
  // default: "<evaluation-dataset-display-name> run - <timestamp>".
  string display_name = 2 [(google.api.field_behavior) = OPTIONAL];

  // Output only. The evaluation results that are part of this run.
  // Format:
  // `projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation}/results/{result}`
  repeated string evaluation_results = 3 [
    (google.api.field_behavior) = OUTPUT_ONLY,
    (google.api.resource_reference) = {
      type: "ces.googleapis.com/EvaluationResult"
    }
  ];

  // Output only. Timestamp when the evaluation run was created.
  google.protobuf.Timestamp create_time = 4
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The user who initiated the evaluation run.
  string initiated_by = 5 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The app version to evaluate.
  // Format:
  // `projects/{project}/locations/{location}/apps/{app}/versions/{version}`
  string app_version = 6 [
    (google.api.field_behavior) = OUTPUT_ONLY,
    (google.api.resource_reference) = { type: "ces.googleapis.com/AppVersion" }
  ];

  // Output only. The display name of the `app_version` that the evaluation ran
  // against.
  string app_version_display_name = 13
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The changelog of the app version that the evaluation ran
  // against. This is populated if user runs evaluation on latest/draft.
  string changelog = 22 [
    (google.api.field_behavior) = OUTPUT_ONLY,
    (google.api.resource_reference) = { type: "ces.googleapis.com/Changelog" }
  ];

  // Output only. The create time of the changelog of the app version that the
  // evaluation ran against. This is populated if user runs evaluation on
  // latest/draft.
  google.protobuf.Timestamp changelog_create_time = 23
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The evaluations that are part of this run. The list may
  // contain evaluations of either type. This field is mutually exclusive with
  // `evaluation_dataset`.
  // Format:
  // `projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation}`
  repeated string evaluations = 7 [
    (google.api.field_behavior) = OUTPUT_ONLY,
    (google.api.resource_reference) = { type: "ces.googleapis.com/Evaluation" }
  ];

  // Output only. The evaluation dataset that this run is associated with. This
  // field is mutually exclusive with `evaluations`. Format:
  // `projects/{project}/locations/{location}/apps/{app}/evaluationDatasets/{evaluationDataset}`
  string evaluation_dataset = 8 [
    (google.api.field_behavior) = OUTPUT_ONLY,
    (google.api.resource_reference) = {
      type: "ces.googleapis.com/EvaluationDataset"
    }
  ];

  // Output only. The type of the evaluations in this run.
  EvaluationType evaluation_type = 9
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The state of the evaluation run.
  EvaluationRunState state = 10 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The progress of the evaluation run.
  Progress progress = 11 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The configuration used in the run.
  EvaluationConfig config = 12 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Deprecated: Use error_info instead. Errors encountered during
  // execution.
  google.rpc.Status error = 14
      [deprecated = true, (google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Error information for the evaluation run.
  EvaluationErrorInfo error_info = 17
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Map of evaluation name to EvaluationRunSummary.
  map<string, EvaluationRunSummary> evaluation_run_summaries = 15
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Latency report for the evaluation run.
  LatencyReport latency_report = 25 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The number of times the evaluations inside the run were run.
  int32 run_count = 16 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The configuration to use for the run per persona.
  repeated PersonaRunConfig persona_run_configs = 18
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Optional. Configuration for running the optimization step after the
  // evaluation run. If not set, the optimization step will not be run.
  OptimizationConfig optimization_config = 19
      [(google.api.field_behavior) = OPTIONAL];

  // Output only. The scheduled evaluation run resource name that created this
  // evaluation run. This field is only set if the evaluation run was created
  // by a scheduled evaluation run.
  // Format:
  // `projects/{project}/locations/{location}/apps/{app}/scheduledEvaluationRuns/{scheduled_evaluation_run}`
  string scheduled_evaluation_run = 20 [
    (google.api.field_behavior) = OUTPUT_ONLY,
    (google.api.resource_reference) = {
      type: "ces.googleapis.com/ScheduledEvaluationRun"
    }
  ];

  // Output only. The method used to run the evaluation.
  GoldenRunMethod golden_run_method = 21
      [(google.api.field_behavior) = OUTPUT_ONLY];
}

// Latency report for the evaluation run.
message LatencyReport {
  // Latency metrics for a component.
  message LatencyMetrics {
    // Output only. The 50th percentile latency.
    google.protobuf.Duration p50_latency = 1
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The 90th percentile latency.
    google.protobuf.Duration p90_latency = 2
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The 99th percentile latency.
    google.protobuf.Duration p99_latency = 3
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The number of times the resource was called.
    int32 call_count = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // Latency metrics for a single tool.
  message ToolLatency {
    // The identifier of the tool.
    oneof tool_identifier {
      // Output only. Format:
      // `projects/{project}/locations/{location}/apps/{app}/tools/{tool}`.
      string tool = 1 [
        (google.api.field_behavior) = OUTPUT_ONLY,
        (google.api.resource_reference) = { type: "ces.googleapis.com/Tool" }
      ];

      // Output only. The toolset tool identifier.
      ToolsetTool toolset_tool = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
    }

    // Output only. The display name of the tool.
    string tool_display_name = 3 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The latency metrics for the tool.
    LatencyMetrics latency_metrics = 4
        [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // Latency metrics for a single callback.
  message CallbackLatency {
    // Output only. The stage of the callback.
    string stage = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The latency metrics for the callback.
    LatencyMetrics latency_metrics = 2
        [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // Latency metrics for a single guardrail.
  message GuardrailLatency {
    // Output only. The name of the guardrail.
    // Format:
    // `projects/{project}/locations/{location}/apps/{app}/guardrails/{guardrail}`.
    string guardrail = 1 [
      (google.api.field_behavior) = OUTPUT_ONLY,
      (google.api.resource_reference) = { type: "ces.googleapis.com/Guardrail" }
    ];

    // Output only. The display name of the guardrail.
    string guardrail_display_name = 2
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The latency metrics for the guardrail.
    LatencyMetrics latency_metrics = 3
        [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // Latency metrics for a single LLM call.
  message LlmCallLatency {
    // Output only. The name of the model.
    string model = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The latency metrics for the LLM call.
    LatencyMetrics latency_metrics = 2
        [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // Output only. Unordered list. Latency metrics for each tool.
  repeated ToolLatency tool_latencies = 1 [
    (google.api.field_behavior) = OUTPUT_ONLY,
    (google.api.field_behavior) = UNORDERED_LIST
  ];

  // Output only. Unordered list. Latency metrics for each callback.
  repeated CallbackLatency callback_latencies = 2 [
    (google.api.field_behavior) = OUTPUT_ONLY,
    (google.api.field_behavior) = UNORDERED_LIST
  ];

  // Output only. Unordered list. Latency metrics for each guardrail.
  repeated GuardrailLatency guardrail_latencies = 3 [
    (google.api.field_behavior) = OUTPUT_ONLY,
    (google.api.field_behavior) = UNORDERED_LIST
  ];

  // Output only. Unordered list. Latency metrics for each LLM call.
  repeated LlmCallLatency llm_call_latencies = 4 [
    (google.api.field_behavior) = OUTPUT_ONLY,
    (google.api.field_behavior) = UNORDERED_LIST
  ];

  // Output only. The total number of sessions considered in the latency report.
  int32 session_count = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// An evaluation expectation represents a specific criteria to evaluate against.
message EvaluationExpectation {
  option (google.api.resource) = {
    type: "ces.googleapis.com/EvaluationExpectation"
    pattern: "projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluation_expectation}"
    plural: "evaluationExpectations"
    singular: "evaluationExpectation"
  };

  // Configuration for LLM-based evaluation criteria.
  message LlmCriteria {
    // Required. The prompt/instructions provided to the LLM judge.
    string prompt = 1 [(google.api.field_behavior) = REQUIRED];
  }

  // The definition of the criteria used for evaluation.
  oneof criteria {
    // Optional. Evaluation criteria based on an LLM prompt.
    LlmCriteria llm_criteria = 3 [(google.api.field_behavior) = OPTIONAL];
  }

  // Identifier. The unique identifier of this evaluation expectation.
  // Format:
  // `projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluation_expectation}`
  string name = 1 [(google.api.field_behavior) = IDENTIFIER];

  // Required. User-defined display name. Must be unique within the app.
  string display_name = 2 [(google.api.field_behavior) = REQUIRED];

  // Optional. User-defined tags for expectations. Can be used to filter
  // expectations.
  repeated string tags = 8 [(google.api.field_behavior) = OPTIONAL];

  // Output only. Timestamp when the evaluation expectation was created.
  google.protobuf.Timestamp create_time = 4
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Timestamp when the evaluation expectation was last updated.
  google.protobuf.Timestamp update_time = 5
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Etag used to ensure the object hasn't changed during a
  // read-modify-write operation. If the etag is empty, the update will
  // overwrite any concurrent changes.
  string etag = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// EvaluationConfig configures settings for running the evaluation.
message EvaluationConfig {
  // The channel to evaluate.
  enum EvaluationChannel {
    // Unspecified evaluation channel.
    EVALUATION_CHANNEL_UNSPECIFIED = 0;

    // Text-only evaluation channel.
    TEXT = 1;

    // Audio evaluation channel.
    AUDIO = 2;
  }

  // Optional. Configuration for processing the input audio.
  InputAudioConfig input_audio_config = 1
      [deprecated = true, (google.api.field_behavior) = OPTIONAL];

  // Optional. Configuration for generating the output audio.
  OutputAudioConfig output_audio_config = 2
      [deprecated = true, (google.api.field_behavior) = OPTIONAL];

  // Optional. The channel to evaluate.
  EvaluationChannel evaluation_channel = 3
      [(google.api.field_behavior) = OPTIONAL];

  // Optional. Specifies whether the evaluation should use real tool calls or
  // fake tools.
  EvaluationToolCallBehaviour tool_call_behaviour = 4
      [(google.api.field_behavior) = OPTIONAL];
}

// Information about an error encountered during an evaluation execution.
message EvaluationErrorInfo {
  // The type of error
  enum ErrorType {
    // Unspecified error type.
    ERROR_TYPE_UNSPECIFIED = 0;

    // Failure during runtime execution.
    RUNTIME_FAILURE = 1;

    // Failure to retrieve conversation from CES Runtime.
    CONVERSATION_RETRIEVAL_FAILURE = 2;

    // Failure to calculate a metric / outcome.
    METRIC_CALCULATION_FAILURE = 3;

    // Failure to update the evaluation.
    EVALUATION_UPDATE_FAILURE = 4;

    // Ran out of quota.
    QUOTA_EXHAUSTED = 5;

    // Failure during user simulation.
    USER_SIMULATION_FAILURE = 6;
  }

  // Output only. The type of error.
  ErrorType error_type = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The error message.
  string error_message = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The session ID for the conversation that caused the error.
  string session_id = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// Request message for
// [EvaluationService.RunEvaluation][google.cloud.ces.v1beta.EvaluationService.RunEvaluation].
message RunEvaluationRequest {
  // Required. The app to evaluate.
  // Format:
  // `projects/{project}/locations/{location}/apps/{app}`
  string app = 1 [
    (google.api.field_behavior) = REQUIRED,
    (google.api.resource_reference) = { type: "ces.googleapis.com/App" }
  ];

  // Optional. List of evaluations to run.
  // Format:
  // `projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation}`
  repeated string evaluations = 2 [
    (google.api.field_behavior) = OPTIONAL,
    (google.api.resource_reference) = { type: "ces.googleapis.com/Evaluation" }
  ];

  // Optional. An evaluation dataset to run.
  // Format:
  // `projects/{project}/locations/{location}/apps/{app}/evaluationDatasets/{evaluationDataset}`
  string evaluation_dataset = 3 [
    (google.api.field_behavior) = OPTIONAL,
    (google.api.resource_reference) = {
      type: "ces.googleapis.com/EvaluationDataset"
    }
  ];

  // Optional. The display name of the evaluation run.
  string display_name = 4 [(google.api.field_behavior) = OPTIONAL];

  // Optional. The app version to evaluate.
  // Format:
  // `projects/{project}/locations/{location}/apps/{app}/versions/{version}`
  string app_version = 6 [
    (google.api.field_behavior) = OPTIONAL,
    (google.api.resource_reference) = { type: "ces.googleapis.com/AppVersion" }
  ];

  // Optional. The configuration to use for the run.
  EvaluationConfig config = 8 [(google.api.field_behavior) = OPTIONAL];

  // Optional. The number of times to run the evaluation. If not set, the
  // default value is 1 per golden, and 5 per scenario.
  optional int32 run_count = 9 [(google.api.field_behavior) = OPTIONAL];

  // Optional. The configuration to use for the run per persona.
  repeated PersonaRunConfig persona_run_configs = 10
      [(google.api.field_behavior) = OPTIONAL];

  // Optional. Configuration for running the optimization step after the
  // evaluation run. If not set, the optimization step will not be run.
  OptimizationConfig optimization_config = 11
      [(google.api.field_behavior) = OPTIONAL];

  // Optional. The resource name of the `ScheduledEvaluationRun` that is
  // triggering this evaluation run.
  //
  // If this field is set, the `scheduled_evaluation_run` field on the created
  // `EvaluationRun` resource will be populated from this value.
  // Format:
  // `projects/{project}/locations/{location}/apps/{app}/scheduledEvaluationRuns/{scheduled_evaluation_run}`
  string scheduled_evaluation_run = 12 [
    (google.api.field_behavior) = OPTIONAL,
    (google.api.resource_reference) = {
      type: "ces.googleapis.com/ScheduledEvaluationRun"
    }
  ];

  // Optional. The method to run the evaluation if it is a golden evaluation. If
  // not set, default to STABLE.
  GoldenRunMethod golden_run_method = 13
      [(google.api.field_behavior) = OPTIONAL];

  // Optional. Whether to generate a latency report for the evaluation run.
  bool generate_latency_report = 14 [(google.api.field_behavior) = OPTIONAL];
}

// Represents a scheduled evaluation run configuration.
message ScheduledEvaluationRun {
  option (google.api.resource) = {
    type: "ces.googleapis.com/ScheduledEvaluationRun"
    pattern: "projects/{project}/locations/{location}/apps/{app}/scheduledEvaluationRuns/{scheduled_evaluation_run}"
    plural: "scheduledEvaluationRuns"
    singular: "scheduledEvaluationRun"
  };

  // Eval scheduling configuration details
  message SchedulingConfig {
    // The frequencies evals can be run at
    enum Frequency {
      // The frequency is unspecified.
      FREQUENCY_UNSPECIFIED = 0;

      // Indicates a one-time scheduled run that should not repeat
      NONE = 1;

      // Run the evaluation daily.
      DAILY = 2;

      // Run the evaluation weekly.
      WEEKLY = 3;

      // Run the evaluation biweekly.
      BIWEEKLY = 4;
    }

    // Required. The frequency with which to run the eval
    Frequency frequency = 1 [(google.api.field_behavior) = REQUIRED];

    // Required. Timestamp when the eval should start.
    google.protobuf.Timestamp start_time = 2
        [(google.api.field_behavior) = REQUIRED];

    // Optional. The days of the week to run the eval. Applicable only for
    // Weekly and Biweekly frequencies. 1 is Monday, 2 is Tuesday, ..., 7 is
    // Sunday.
    repeated int32 days_of_week = 3 [(google.api.field_behavior) = OPTIONAL];
  }

  // Identifier. The unique identifier of the scheduled evaluation run config.
  // Format:
  // projects/{projectId}/locations/{locationId}/apps/{appId}/scheduledEvaluationRuns/{scheduledEvaluationRunId}
  string name = 1 [(google.api.field_behavior) = IDENTIFIER];

  // Required. User-defined display name of the scheduled evaluation run config.
  string display_name = 2 [(google.api.field_behavior) = REQUIRED];

  // Required. The RunEvaluationRequest to schedule
  RunEvaluationRequest request = 3 [(google.api.field_behavior) = REQUIRED];

  // Optional. User-defined description of the scheduled evaluation run.
  string description = 4 [(google.api.field_behavior) = OPTIONAL];

  // Required. Configuration for the timing and frequency with which to execute
  // the evaluations.
  SchedulingConfig scheduling_config = 5
      [(google.api.field_behavior) = REQUIRED];

  // Optional. Whether this config is active
  bool active = 6 [(google.api.field_behavior) = OPTIONAL];

  // Output only. The last successful EvaluationRun of this scheduled execution.
  // Format:
  // `projects/{project}/locations/{location}/apps/{app}/evaluationRuns/{evaluationRun}`
  string last_completed_run = 7 [
    (google.api.field_behavior) = OUTPUT_ONLY,
    (google.api.resource_reference) = {
      type: "ces.googleapis.com/EvaluationRun"
    }
  ];

  // Output only. The total number of times this run has been executed
  int32 total_executions = 8 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The next time this is scheduled to execute
  google.protobuf.Timestamp next_scheduled_execution_time = 9
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Timestamp when the scheduled evaluation run was created.
  google.protobuf.Timestamp create_time = 10
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The user who created the scheduled evaluation run.
  string created_by = 11 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Timestamp when the evaluation was last updated.
  google.protobuf.Timestamp update_time = 12
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The user who last updated the evaluation.
  string last_updated_by = 13 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Etag used to ensure the object hasn't changed during a
  // read-modify-write operation. If the etag is empty, the update will
  // overwrite any concurrent changes.
  string etag = 14 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// Configuration for running an evaluation for a specific persona.
message PersonaRunConfig {
  // Optional. The persona to use for the evaluation.
  // Format:
  // `projects/{project}/locations/{location}/apps/{app}/evaluationPersonas/{evaluationPersona}`
  string persona = 1 [(google.api.field_behavior) = OPTIONAL];

  // Optional. The number of tasks to run for the persona.
  int32 task_count = 2 [(google.api.field_behavior) = OPTIONAL];
}

// Configuration for running the optimization step after the evaluation run.
message OptimizationConfig {
  // The status of the optimization run.
  enum OptimizationStatus {
    // Optimization status is not specified.
    OPTIMIZATION_STATUS_UNSPECIFIED = 0;

    // Optimization is running.
    RUNNING = 1;

    // Optimization has completed.
    COMPLETED = 2;

    // Optimization failed due to an internal error.
    ERROR = 3;
  }

  // Optional. Whether to generate a loss report.
  bool generate_loss_report = 1 [(google.api.field_behavior) = OPTIONAL];

  // Output only. The assistant session to use for the optimization based on
  // this evaluation run. Format:
  // `projects/{project}/locations/{location}/apps/{app}/assistantSessions/{assistantSession}`
  string assistant_session = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The summary of the loss report.
  string report_summary = 3 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Whether to suggest a fix for the losses.
  bool should_suggest_fix = 5 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The status of the optimization run.
  OptimizationStatus status = 4 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The error message if the optimization run failed.
  string error_message = 6 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The generated loss report.
  google.protobuf.Struct loss_report = 7
      [(google.api.field_behavior) = OUTPUT_ONLY];
}