googleapis/google/cloud/aiplatform/v1/content.proto

// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package google.cloud.aiplatform.v1;

import "google/api/field_behavior.proto";
import "google/cloud/aiplatform/v1/openapi.proto";
import "google/cloud/aiplatform/v1/tool.proto";
import "google/cloud/aiplatform/v1/vertex_rag_data.proto";
import "google/protobuf/duration.proto";
import "google/type/date.proto";

option csharp_namespace = "Google.Cloud.AIPlatform.V1";
option go_package = "cloud.google.com/go/aiplatform/apiv1/aiplatformpb;aiplatformpb";
option java_multiple_files = true;
option java_outer_classname = "ContentProto";
option java_package = "com.google.cloud.aiplatform.v1";
option php_namespace = "Google\\Cloud\\AIPlatform\\V1";
option ruby_package = "Google::Cloud::AIPlatform::V1";

// Harm categories that will block the content.
enum HarmCategory {
  // The harm category is unspecified.
  HARM_CATEGORY_UNSPECIFIED = 0;

  // The harm category is hate speech.
  HARM_CATEGORY_HATE_SPEECH = 1;

  // The harm category is dangerous content.
  HARM_CATEGORY_DANGEROUS_CONTENT = 2;

  // The harm category is harassment.
  HARM_CATEGORY_HARASSMENT = 3;

  // The harm category is sexually explicit content.
  HARM_CATEGORY_SEXUALLY_EXPLICIT = 4;

  // Deprecated: Election filter is not longer supported.
  // The harm category is civic integrity.
  HARM_CATEGORY_CIVIC_INTEGRITY = 5 [deprecated = true];
}

// Content Part modality
enum Modality {
  // Unspecified modality.
  MODALITY_UNSPECIFIED = 0;

  // Plain text.
  TEXT = 1;

  // Image.
  IMAGE = 2;

  // Video.
  VIDEO = 3;

  // Audio.
  AUDIO = 4;

  // Document, e.g. PDF.
  DOCUMENT = 5;
}

// The base structured datatype containing multi-part content of a message.
//
// A `Content` includes a `role` field designating the producer of the `Content`
// and a `parts` field containing multi-part data that contains the content of
// the message turn.
message Content {
  // Optional. The producer of the content. Must be either 'user' or 'model'.
  //
  // Useful to set for multi-turn conversations, otherwise can be left blank
  // or unset.
  string role = 1 [(google.api.field_behavior) = OPTIONAL];

  // Required. Ordered `Parts` that constitute a single message. Parts may have
  // different IANA MIME types.
  repeated Part parts = 2 [(google.api.field_behavior) = REQUIRED];
}

// A datatype containing media that is part of a multi-part `Content` message.
//
// A `Part` consists of data which has an associated datatype. A `Part` can only
// contain one of the accepted types in `Part.data`.
//
// A `Part` must have a fixed IANA MIME type identifying the type and subtype
// of the media if `inline_data` or `file_data` field is filled with raw bytes.
message Part {
  oneof data {
    // Optional. Text part (can be code).
    string text = 1 [(google.api.field_behavior) = OPTIONAL];

    // Optional. Inlined bytes data.
    Blob inline_data = 2 [(google.api.field_behavior) = OPTIONAL];

    // Optional. URI based data.
    FileData file_data = 3 [(google.api.field_behavior) = OPTIONAL];

    // Optional. A predicted [FunctionCall] returned from the model that
    // contains a string representing the [FunctionDeclaration.name] with the
    // parameters and their values.
    FunctionCall function_call = 5 [(google.api.field_behavior) = OPTIONAL];

    // Optional. The result output of a [FunctionCall] that contains a string
    // representing the [FunctionDeclaration.name] and a structured JSON object
    // containing any output from the function call. It is used as context to
    // the model.
    FunctionResponse function_response = 6
        [(google.api.field_behavior) = OPTIONAL];

    // Optional. Code generated by the model that is meant to be executed.
    ExecutableCode executable_code = 8 [(google.api.field_behavior) = OPTIONAL];

    // Optional. Result of executing the [ExecutableCode].
    CodeExecutionResult code_execution_result = 9
        [(google.api.field_behavior) = OPTIONAL];

    // Indicates if the part is thought from the model.
    bool thought = 10 [(google.api.field_behavior) = OPTIONAL];

    // An opaque signature for the thought so it can be reused in subsequent
    // requests.
    bytes thought_signature = 11 [(google.api.field_behavior) = OPTIONAL];
  }

  oneof metadata {
    // Optional. Video metadata. The metadata should only be specified while the
    // video data is presented in inline_data or file_data.
    VideoMetadata video_metadata = 4 [(google.api.field_behavior) = OPTIONAL];
  }
}

// Content blob.
//
// It's preferred to send as [text][google.cloud.aiplatform.v1.Part.text]
// directly rather than raw bytes.
message Blob {
  // Required. The IANA standard MIME type of the source data.
  string mime_type = 1 [(google.api.field_behavior) = REQUIRED];

  // Required. Raw bytes.
  bytes data = 2 [(google.api.field_behavior) = REQUIRED];
}

// URI based data.
message FileData {
  // Required. The IANA standard MIME type of the source data.
  string mime_type = 1 [(google.api.field_behavior) = REQUIRED];

  // Required. URI.
  string file_uri = 2 [(google.api.field_behavior) = REQUIRED];
}

// Metadata describes the input video content.
message VideoMetadata {
  // Optional. The start offset of the video.
  google.protobuf.Duration start_offset = 1
      [(google.api.field_behavior) = OPTIONAL];

  // Optional. The end offset of the video.
  google.protobuf.Duration end_offset = 2
      [(google.api.field_behavior) = OPTIONAL];
}

// Generation config.
message GenerationConfig {
  // The configuration for routing the request to a specific model.
  message RoutingConfig {
    // When automated routing is specified, the routing will be determined by
    // the pretrained routing model and customer provided model routing
    // preference.
    message AutoRoutingMode {
      // The model routing preference.
      enum ModelRoutingPreference {
        // Unspecified model routing preference.
        UNKNOWN = 0;

        // Prefer higher quality over low cost.
        PRIORITIZE_QUALITY = 1;

        // Balanced model routing preference.
        BALANCED = 2;

        // Prefer lower cost over higher quality.
        PRIORITIZE_COST = 3;
      }

      // The model routing preference.
      optional ModelRoutingPreference model_routing_preference = 1;
    }

    // When manual routing is set, the specified model will be used directly.
    message ManualRoutingMode {
      // The model name to use. Only the public LLM models are accepted. e.g.
      // 'gemini-1.5-pro-001'.
      optional string model_name = 1;
    }

    // Routing mode.
    oneof routing_config {
      // Automated routing.
      AutoRoutingMode auto_mode = 1;

      // Manual routing.
      ManualRoutingMode manual_mode = 2;
    }
  }

  // Config for thinking features.
  message ThinkingConfig {
    // Indicates whether to include thoughts in the response.
    // If true, thoughts are returned only when available.
    optional bool include_thoughts = 1 [(google.api.field_behavior) = OPTIONAL];

    // Optional. Indicates the thinking budget in tokens.
    // This is only applied when enable_thinking is true.
    optional int32 thinking_budget = 3 [(google.api.field_behavior) = OPTIONAL];
  }

  // Optional. Controls the randomness of predictions.
  optional float temperature = 1 [(google.api.field_behavior) = OPTIONAL];

  // Optional. If specified, nucleus sampling will be used.
  optional float top_p = 2 [(google.api.field_behavior) = OPTIONAL];

  // Optional. If specified, top-k sampling will be used.
  optional float top_k = 3 [(google.api.field_behavior) = OPTIONAL];

  // Optional. Number of candidates to generate.
  optional int32 candidate_count = 4 [(google.api.field_behavior) = OPTIONAL];

  // Optional. The maximum number of output tokens to generate per message.
  optional int32 max_output_tokens = 5 [(google.api.field_behavior) = OPTIONAL];

  // Optional. Stop sequences.
  repeated string stop_sequences = 6 [(google.api.field_behavior) = OPTIONAL];

  // Optional. If true, export the logprobs results in response.
  optional bool response_logprobs = 18 [(google.api.field_behavior) = OPTIONAL];

  // Optional. Logit probabilities.
  optional int32 logprobs = 7 [(google.api.field_behavior) = OPTIONAL];

  // Optional. Positive penalties.
  optional float presence_penalty = 8 [(google.api.field_behavior) = OPTIONAL];

  // Optional. Frequency penalties.
  optional float frequency_penalty = 9 [(google.api.field_behavior) = OPTIONAL];

  // Optional. Seed.
  optional int32 seed = 12 [(google.api.field_behavior) = OPTIONAL];

  // Optional. Output response mimetype of the generated candidate text.
  // Supported mimetype:
  // - `text/plain`: (default) Text output.
  // - `application/json`: JSON response in the candidates.
  // The model needs to be prompted to output the appropriate response type,
  // otherwise the behavior is undefined.
  // This is a preview feature.
  string response_mime_type = 13 [(google.api.field_behavior) = OPTIONAL];

  // Optional. The `Schema` object allows the definition of input and output
  // data types. These types can be objects, but also primitives and arrays.
  // Represents a select subset of an [OpenAPI 3.0 schema
  // object](https://spec.openapis.org/oas/v3.0.3#schema).
  // If set, a compatible response_mime_type must also be set.
  // Compatible mimetypes:
  // `application/json`: Schema for JSON response.
  optional Schema response_schema = 16 [(google.api.field_behavior) = OPTIONAL];

  // Optional. Routing configuration.
  optional RoutingConfig routing_config = 17
      [(google.api.field_behavior) = OPTIONAL];

  // Optional. Config for thinking features.
  // An error will be returned if this field is set for models that don't
  // support thinking.
  ThinkingConfig thinking_config = 25 [(google.api.field_behavior) = OPTIONAL];
}

// Safety settings.
message SafetySetting {
  // Probability based thresholds levels for blocking.
  enum HarmBlockThreshold {
    // Unspecified harm block threshold.
    HARM_BLOCK_THRESHOLD_UNSPECIFIED = 0;

    // Block low threshold and above (i.e. block more).
    BLOCK_LOW_AND_ABOVE = 1;

    // Block medium threshold and above.
    BLOCK_MEDIUM_AND_ABOVE = 2;

    // Block only high threshold (i.e. block less).
    BLOCK_ONLY_HIGH = 3;

    // Block none.
    BLOCK_NONE = 4;

    // Turn off the safety filter.
    OFF = 5;
  }

  // Probability vs severity.
  enum HarmBlockMethod {
    // The harm block method is unspecified.
    HARM_BLOCK_METHOD_UNSPECIFIED = 0;

    // The harm block method uses both probability and severity scores.
    SEVERITY = 1;

    // The harm block method uses the probability score.
    PROBABILITY = 2;
  }

  // Required. Harm category.
  HarmCategory category = 1 [(google.api.field_behavior) = REQUIRED];

  // Required. The harm block threshold.
  HarmBlockThreshold threshold = 2 [(google.api.field_behavior) = REQUIRED];

  // Optional. Specify if the threshold is used for probability or severity
  // score. If not specified, the threshold is used for probability score.
  HarmBlockMethod method = 4 [(google.api.field_behavior) = OPTIONAL];
}

// Safety rating corresponding to the generated content.
message SafetyRating {
  // Harm probability levels in the content.
  enum HarmProbability {
    // Harm probability unspecified.
    HARM_PROBABILITY_UNSPECIFIED = 0;

    // Negligible level of harm.
    NEGLIGIBLE = 1;

    // Low level of harm.
    LOW = 2;

    // Medium level of harm.
    MEDIUM = 3;

    // High level of harm.
    HIGH = 4;
  }

  // Harm severity levels.
  enum HarmSeverity {
    // Harm severity unspecified.
    HARM_SEVERITY_UNSPECIFIED = 0;

    // Negligible level of harm severity.
    HARM_SEVERITY_NEGLIGIBLE = 1;

    // Low level of harm severity.
    HARM_SEVERITY_LOW = 2;

    // Medium level of harm severity.
    HARM_SEVERITY_MEDIUM = 3;

    // High level of harm severity.
    HARM_SEVERITY_HIGH = 4;
  }

  // Output only. Harm category.
  HarmCategory category = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Harm probability levels in the content.
  HarmProbability probability = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Harm probability score.
  float probability_score = 5 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Harm severity levels in the content.
  HarmSeverity severity = 6 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Harm severity score.
  float severity_score = 7 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Indicates whether the content was filtered out because of this
  // rating.
  bool blocked = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// A collection of source attributions for a piece of content.
message CitationMetadata {
  // Output only. List of citations.
  repeated Citation citations = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// Source attributions for content.
message Citation {
  // Output only. Start index into the content.
  int32 start_index = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. End index into the content.
  int32 end_index = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Url reference of the attribution.
  string uri = 3 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Title of the attribution.
  string title = 4 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. License of the attribution.
  string license = 5 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Publication date of the attribution.
  google.type.Date publication_date = 6
      [(google.api.field_behavior) = OUTPUT_ONLY];
}

// A response candidate generated from the model.
message Candidate {
  // The reason why the model stopped generating tokens.
  // If empty, the model has not stopped generating the tokens.
  enum FinishReason {
    // The finish reason is unspecified.
    FINISH_REASON_UNSPECIFIED = 0;

    // Token generation reached a natural stopping point or a configured stop
    // sequence.
    STOP = 1;

    // Token generation reached the configured maximum output tokens.
    MAX_TOKENS = 2;

    // Token generation stopped because the content potentially contains safety
    // violations. NOTE: When streaming,
    // [content][google.cloud.aiplatform.v1.Candidate.content] is empty if
    // content filters blocks the output.
    SAFETY = 3;

    // Token generation stopped because the content potentially contains
    // copyright violations.
    RECITATION = 4;

    // All other reasons that stopped the token generation.
    OTHER = 5;

    // Token generation stopped because the content contains forbidden terms.
    BLOCKLIST = 6;

    // Token generation stopped for potentially containing prohibited content.
    PROHIBITED_CONTENT = 7;

    // Token generation stopped because the content potentially contains
    // Sensitive Personally Identifiable Information (SPII).
    SPII = 8;

    // The function call generated by the model is invalid.
    MALFORMED_FUNCTION_CALL = 9;
  }

  // Output only. Index of the candidate.
  int32 index = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Content parts of the candidate.
  Content content = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Confidence score of the candidate.
  double score = 8 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Average log probability score of the candidate.
  double avg_logprobs = 9 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Log-likelihood scores for the response tokens and top tokens
  LogprobsResult logprobs_result = 10
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The reason why the model stopped generating tokens.
  // If empty, the model has not stopped generating the tokens.
  FinishReason finish_reason = 3 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. List of ratings for the safety of a response candidate.
  //
  // There is at most one rating per category.
  repeated SafetyRating safety_ratings = 4
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Describes the reason the mode stopped generating tokens in
  // more detail. This is only filled when `finish_reason` is set.
  optional string finish_message = 5
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Source attribution of the generated content.
  CitationMetadata citation_metadata = 6
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Metadata specifies sources used to ground generated content.
  GroundingMetadata grounding_metadata = 7
      [(google.api.field_behavior) = OUTPUT_ONLY];
}

// Logprobs Result
message LogprobsResult {
  // Candidate for the logprobs token and score.
  message Candidate {
    // The candidate’s token string value.
    optional string token = 1;

    // The candidate’s token id value.
    optional int32 token_id = 3;

    // The candidate's log probability.
    optional float log_probability = 2;
  }

  // Candidates with top log probabilities at each decoding step.
  message TopCandidates {
    // Sorted by log probability in descending order.
    repeated Candidate candidates = 1;
  }

  // Length = total number of decoding steps.
  repeated TopCandidates top_candidates = 1;

  // Length = total number of decoding steps.
  // The chosen candidates may or may not be in top_candidates.
  repeated Candidate chosen_candidates = 2;
}

// Segment of the content.
message Segment {
  // Output only. The index of a Part object within its parent Content object.
  int32 part_index = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Start index in the given Part, measured in bytes. Offset from
  // the start of the Part, inclusive, starting at zero.
  int32 start_index = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. End index in the given Part, measured in bytes. Offset from
  // the start of the Part, exclusive, starting at zero.
  int32 end_index = 3 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The text corresponding to the segment from the response.
  string text = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// Grounding chunk.
message GroundingChunk {
  // Chunk from the web.
  message Web {
    // URI reference of the chunk.
    optional string uri = 1;

    // Title of the chunk.
    optional string title = 2;
  }

  // Chunk from context retrieved by the retrieval tools.
  message RetrievedContext {
    // Tool-specific details about the retrieved context.
    oneof context_details {
      // Additional context for the RAG retrieval result. This is only populated
      // when using the RAG retrieval tool.
      RagChunk rag_chunk = 4;
    }

    // URI reference of the attribution.
    optional string uri = 1;

    // Title of the attribution.
    optional string title = 2;

    // Text of the attribution.
    optional string text = 3;
  }

  // Chunk type.
  oneof chunk_type {
    // Grounding chunk from the web.
    Web web = 1;

    // Grounding chunk from context retrieved by the retrieval tools.
    RetrievedContext retrieved_context = 2;
  }
}

// Grounding support.
message GroundingSupport {
  // Segment of the content this support belongs to.
  optional Segment segment = 1;

  // A list of indices (into 'grounding_chunk') specifying the
  // citations associated with the claim. For instance [1,3,4] means
  // that grounding_chunk[1], grounding_chunk[3],
  // grounding_chunk[4] are the retrieved content attributed to the claim.
  repeated int32 grounding_chunk_indices = 2;

  // Confidence score of the support references. Ranges from 0 to 1. 1 is the
  // most confident. This list must have the same size as the
  // grounding_chunk_indices.
  repeated float confidence_scores = 3;
}

// Metadata returned to client when grounding is enabled.
message GroundingMetadata {
  // Optional. Web search queries for the following-up web search.
  repeated string web_search_queries = 1
      [(google.api.field_behavior) = OPTIONAL];

  // Optional. Google search entry for the following-up web searches.
  optional SearchEntryPoint search_entry_point = 4
      [(google.api.field_behavior) = OPTIONAL];

  // List of supporting references retrieved from specified grounding source.
  repeated GroundingChunk grounding_chunks = 5;

  // Optional. List of grounding support.
  repeated GroundingSupport grounding_supports = 6
      [(google.api.field_behavior) = OPTIONAL];

  // Optional. Output only. Retrieval metadata.
  optional RetrievalMetadata retrieval_metadata = 7 [
    (google.api.field_behavior) = OPTIONAL,
    (google.api.field_behavior) = OUTPUT_ONLY
  ];
}

// Google search entry point.
message SearchEntryPoint {
  // Optional. Web content snippet that can be embedded in a web page or an app
  // webview.
  string rendered_content = 1 [(google.api.field_behavior) = OPTIONAL];

  // Optional. Base64 encoded JSON representing array of <search term, search
  // url> tuple.
  bytes sdk_blob = 2 [(google.api.field_behavior) = OPTIONAL];
}

// Metadata related to retrieval in the grounding flow.
message RetrievalMetadata {
  // Optional. Score indicating how likely information from Google Search could
  // help answer the prompt. The score is in the range `[0, 1]`, where 0 is the
  // least likely and 1 is the most likely. This score is only populated when
  // Google Search grounding and dynamic retrieval is enabled. It will be
  // compared to the threshold to determine whether to trigger Google Search.
  float google_search_dynamic_retrieval_score = 2
      [(google.api.field_behavior) = OPTIONAL];
}

// Represents token counting info for a single modality.
message ModalityTokenCount {
  // The modality associated with this token count.
  Modality modality = 1;

  // Number of tokens.
  int32 token_count = 2;
}