googleapis/google/ai/generativelanguage/v1alpha/content.proto

// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package google.ai.generativelanguage.v1alpha;

import "google/api/field_behavior.proto";
import "google/protobuf/struct.proto";

option go_package = "cloud.google.com/go/ai/generativelanguage/apiv1alpha/generativelanguagepb;generativelanguagepb";
option java_multiple_files = true;
option java_outer_classname = "ContentProto";
option java_package = "com.google.ai.generativelanguage.v1alpha";

// Type contains the list of OpenAPI data types as defined by
// https://spec.openapis.org/oas/v3.0.3#data-types
enum Type {
  // Not specified, should not be used.
  TYPE_UNSPECIFIED = 0;

  // String type.
  STRING = 1;

  // Number type.
  NUMBER = 2;

  // Integer type.
  INTEGER = 3;

  // Boolean type.
  BOOLEAN = 4;

  // Array type.
  ARRAY = 5;

  // Object type.
  OBJECT = 6;
}

// The base structured datatype containing multi-part content of a message.
//
// A `Content` includes a `role` field designating the producer of the `Content`
// and a `parts` field containing multi-part data that contains the content of
// the message turn.
message Content {
  // Ordered `Parts` that constitute a single message. Parts may have different
  // MIME types.
  repeated Part parts = 1;

  // Optional. The producer of the content. Must be either 'user' or 'model'.
  //
  // Useful to set for multi-turn conversations, otherwise can be left blank
  // or unset.
  string role = 2 [(google.api.field_behavior) = OPTIONAL];
}

// A datatype containing media that is part of a multi-part `Content` message.
//
// A `Part` consists of data which has an associated datatype. A `Part` can only
// contain one of the accepted types in `Part.data`.
//
// A `Part` must have a fixed IANA MIME type identifying the type and subtype
// of the media if the `inline_data` field is filled with raw bytes.
message Part {
  oneof data {
    // Inline text.
    string text = 2;

    // Inline media bytes.
    Blob inline_data = 3;

    // A predicted `FunctionCall` returned from the model that contains
    // a string representing the `FunctionDeclaration.name` with the
    // arguments and their values.
    FunctionCall function_call = 4;

    // The result output of a `FunctionCall` that contains a string
    // representing the `FunctionDeclaration.name` and a structured JSON
    // object containing any output from the function is used as context to
    // the model.
    FunctionResponse function_response = 5;

    // URI based data.
    FileData file_data = 6;

    // Code generated by the model that is meant to be executed.
    ExecutableCode executable_code = 9;

    // Result of executing the `ExecutableCode`.
    CodeExecutionResult code_execution_result = 10;
  }
}

// Raw media bytes.
//
// Text should not be sent as raw bytes, use the 'text' field.
message Blob {
  // The IANA standard MIME type of the source data.
  // Examples:
  //   - image/png
  //   - image/jpeg
  // If an unsupported MIME type is provided, an error will be returned. For a
  // complete list of supported types, see [Supported file
  // formats](https://ai.google.dev/gemini-api/docs/prompting_with_media#supported_file_formats).
  string mime_type = 1;

  // Raw bytes for media formats.
  bytes data = 2;
}

// URI based data.
message FileData {
  // Optional. The IANA standard MIME type of the source data.
  string mime_type = 1 [(google.api.field_behavior) = OPTIONAL];

  // Required. URI.
  string file_uri = 2 [(google.api.field_behavior) = REQUIRED];
}

// Code generated by the model that is meant to be executed, and the result
// returned to the model.
//
// Only generated when using the `CodeExecution` tool, in which the code will
// be automatically executed, and a corresponding `CodeExecutionResult` will
// also be generated.
message ExecutableCode {
  // Supported programming languages for the generated code.
  enum Language {
    // Unspecified language. This value should not be used.
    LANGUAGE_UNSPECIFIED = 0;

    // Python >= 3.10, with numpy and simpy available.
    PYTHON = 1;
  }

  // Required. Programming language of the `code`.
  Language language = 1 [(google.api.field_behavior) = REQUIRED];

  // Required. The code to be executed.
  string code = 2 [(google.api.field_behavior) = REQUIRED];
}

// Result of executing the `ExecutableCode`.
//
// Only generated when using the `CodeExecution`, and always follows a `part`
// containing the `ExecutableCode`.
message CodeExecutionResult {
  // Enumeration of possible outcomes of the code execution.
  enum Outcome {
    // Unspecified status. This value should not be used.
    OUTCOME_UNSPECIFIED = 0;

    // Code execution completed successfully.
    OUTCOME_OK = 1;

    // Code execution finished but with a failure. `stderr` should contain the
    // reason.
    OUTCOME_FAILED = 2;

    // Code execution ran for too long, and was cancelled. There may or may not
    // be a partial output present.
    OUTCOME_DEADLINE_EXCEEDED = 3;
  }

  // Required. Outcome of the code execution.
  Outcome outcome = 1 [(google.api.field_behavior) = REQUIRED];

  // Optional. Contains stdout when code execution is successful, stderr or
  // other description otherwise.
  string output = 2 [(google.api.field_behavior) = OPTIONAL];
}

// Tool details that the model may use to generate response.
//
// A `Tool` is a piece of code that enables the system to interact with
// external systems to perform an action, or set of actions, outside of
// knowledge and scope of the model.
message Tool {
  // GoogleSearch tool type.
  // Tool to support Google Search in Model. Powered by Google.
  message GoogleSearch {}

  // Optional. A list of `FunctionDeclarations` available to the model that can
  // be used for function calling.
  //
  // The model or system does not execute the function. Instead the defined
  // function may be returned as a
  // [FunctionCall][google.ai.generativelanguage.v1alpha.Part.function_call]
  // with arguments to the client side for execution. The model may decide to
  // call a subset of these functions by populating
  // [FunctionCall][google.ai.generativelanguage.v1alpha.Part.function_call] in
  // the response. The next conversation turn may contain a
  // [FunctionResponse][google.ai.generativelanguage.v1alpha.Part.function_response]
  // with the [Content.role][google.ai.generativelanguage.v1alpha.Content.role]
  // "function" generation context for the next model turn.
  repeated FunctionDeclaration function_declarations = 1
      [(google.api.field_behavior) = OPTIONAL];

  // Optional. Retrieval tool that is powered by Google search.
  GoogleSearchRetrieval google_search_retrieval = 2
      [(google.api.field_behavior) = OPTIONAL];

  // Optional. Enables the model to execute code as part of generation.
  CodeExecution code_execution = 3 [(google.api.field_behavior) = OPTIONAL];

  // Optional. GoogleSearch tool type.
  // Tool to support Google Search in Model. Powered by Google.
  GoogleSearch google_search = 4 [(google.api.field_behavior) = OPTIONAL];
}

// Tool to retrieve public web data for grounding, powered by Google.
message GoogleSearchRetrieval {
  // Specifies the dynamic retrieval configuration for the given source.
  DynamicRetrievalConfig dynamic_retrieval_config = 1;
}

// Describes the options to customize dynamic retrieval.
message DynamicRetrievalConfig {
  // The mode of the predictor to be used in dynamic retrieval.
  enum Mode {
    // Always trigger retrieval.
    MODE_UNSPECIFIED = 0;

    // Run retrieval only when system decides it is necessary.
    MODE_DYNAMIC = 1;
  }

  // The mode of the predictor to be used in dynamic retrieval.
  Mode mode = 1;

  // The threshold to be used in dynamic retrieval.
  // If not set, a system default value is used.
  optional float dynamic_threshold = 2;
}

// Tool that executes code generated by the model, and automatically returns
// the result to the model.
//
// See also `ExecutableCode` and `CodeExecutionResult` which are only generated
// when using this tool.
message CodeExecution {}

// The Tool configuration containing parameters for specifying `Tool` use
// in the request.
message ToolConfig {
  // Optional. Function calling config.
  FunctionCallingConfig function_calling_config = 1
      [(google.api.field_behavior) = OPTIONAL];
}

// Configuration for specifying function calling behavior.
message FunctionCallingConfig {
  // Defines the execution behavior for function calling by defining the
  // execution mode.
  enum Mode {
    // Unspecified function calling mode. This value should not be used.
    MODE_UNSPECIFIED = 0;

    // Default model behavior, model decides to predict either a function call
    // or a natural language response.
    AUTO = 1;

    // Model is constrained to always predicting a function call only.
    // If "allowed_function_names" are set, the predicted function call will be
    // limited to any one of "allowed_function_names", else the predicted
    // function call will be any one of the provided "function_declarations".
    ANY = 2;

    // Model will not predict any function call. Model behavior is same as when
    // not passing any function declarations.
    NONE = 3;
  }

  // Optional. Specifies the mode in which function calling should execute. If
  // unspecified, the default value will be set to AUTO.
  Mode mode = 1 [(google.api.field_behavior) = OPTIONAL];

  // Optional. A set of function names that, when provided, limits the functions
  // the model will call.
  //
  // This should only be set when the Mode is ANY. Function names
  // should match [FunctionDeclaration.name]. With mode set to ANY, model will
  // predict a function call from the set of function names provided.
  repeated string allowed_function_names = 2
      [(google.api.field_behavior) = OPTIONAL];
}

// Structured representation of a function declaration as defined by the
// [OpenAPI 3.03 specification](https://spec.openapis.org/oas/v3.0.3). Included
// in this declaration are the function name and parameters. This
// FunctionDeclaration is a representation of a block of code that can be used
// as a `Tool` by the model and executed by the client.
message FunctionDeclaration {
  // Required. The name of the function.
  // Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum
  // length of 63.
  string name = 1 [(google.api.field_behavior) = REQUIRED];

  // Required. A brief description of the function.
  string description = 2 [(google.api.field_behavior) = REQUIRED];

  // Optional. Describes the parameters to this function. Reflects the Open
  // API 3.03 Parameter Object string Key: the name of the parameter. Parameter
  // names are case sensitive. Schema Value: the Schema defining the type used
  // for the parameter.
  optional Schema parameters = 3 [(google.api.field_behavior) = OPTIONAL];

  // Optional. Describes the output from this function in JSON Schema format.
  // Reflects the Open API 3.03 Response Object. The Schema defines the type
  // used for the response value of the function.
  optional Schema response = 4 [(google.api.field_behavior) = OPTIONAL];
}

// A predicted `FunctionCall` returned from the model that contains
// a string representing the `FunctionDeclaration.name` with the
// arguments and their values.
message FunctionCall {
  // Optional. The unique id of the function call. If populated, the client to
  // execute the `function_call` and return the response with the matching `id`.
  string id = 3 [(google.api.field_behavior) = OPTIONAL];

  // Required. The name of the function to call.
  // Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum
  // length of 63.
  string name = 1 [(google.api.field_behavior) = REQUIRED];

  // Optional. The function parameters and values in JSON object format.
  optional google.protobuf.Struct args = 2
      [(google.api.field_behavior) = OPTIONAL];
}

// The result output from a `FunctionCall` that contains a string
// representing the `FunctionDeclaration.name` and a structured JSON
// object containing any output from the function is used as context to
// the model. This should contain the result of a`FunctionCall` made
// based on model prediction.
message FunctionResponse {
  // Optional. The id of the function call this response is for. Populated by
  // the client to match the corresponding function call `id`.
  string id = 3 [(google.api.field_behavior) = OPTIONAL];

  // Required. The name of the function to call.
  // Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum
  // length of 63.
  string name = 1 [(google.api.field_behavior) = REQUIRED];

  // Required. The function response in JSON object format.
  google.protobuf.Struct response = 2 [(google.api.field_behavior) = REQUIRED];
}

// The `Schema` object allows the definition of input and output data types.
// These types can be objects, but also primitives and arrays.
// Represents a select subset of an [OpenAPI 3.0 schema
// object](https://spec.openapis.org/oas/v3.0.3#schema).
message Schema {
  // Required. Data type.
  Type type = 1 [(google.api.field_behavior) = REQUIRED];

  // Optional. The format of the data. This is used only for primitive
  // datatypes. Supported formats:
  //  for NUMBER type: float, double
  //  for INTEGER type: int32, int64
  //  for STRING type: enum
  string format = 2 [(google.api.field_behavior) = OPTIONAL];

  // Optional. A brief description of the parameter. This could contain examples
  // of use. Parameter description may be formatted as Markdown.
  string description = 3 [(google.api.field_behavior) = OPTIONAL];

  // Optional. Indicates if the value may be null.
  bool nullable = 4 [(google.api.field_behavior) = OPTIONAL];

  // Optional. Possible values of the element of Type.STRING with enum format.
  // For example we can define an Enum Direction as :
  // {type:STRING, format:enum, enum:["EAST", NORTH", "SOUTH", "WEST"]}
  repeated string enum = 5 [(google.api.field_behavior) = OPTIONAL];

  // Optional. Schema of the elements of Type.ARRAY.
  optional Schema items = 6 [(google.api.field_behavior) = OPTIONAL];

  // Optional. Maximum number of the elements for Type.ARRAY.
  int64 max_items = 21 [(google.api.field_behavior) = OPTIONAL];

  // Optional. Minimum number of the elements for Type.ARRAY.
  int64 min_items = 22 [(google.api.field_behavior) = OPTIONAL];

  // Optional. Properties of Type.OBJECT.
  map<string, Schema> properties = 7 [(google.api.field_behavior) = OPTIONAL];

  // Optional. Required properties of Type.OBJECT.
  repeated string required = 8 [(google.api.field_behavior) = OPTIONAL];
}

// Passage included inline with a grounding configuration.
message GroundingPassage {
  // Identifier for the passage for attributing this passage in grounded
  // answers.
  string id = 1;

  // Content of the passage.
  Content content = 2;
}

// A repeated list of passages.
message GroundingPassages {
  // List of passages.
  repeated GroundingPassage passages = 1;
}