googleapis/google/cloud/aiplatform/v1/tool.proto
Google APIs ebfdba37e5 feat: add VALIDATED Function Calling mode to v1
PiperOrigin-RevId: 885373782
2026-03-17 21:59:55 -07:00

682 lines
26 KiB
Protocol Buffer

// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
package google.cloud.aiplatform.v1;
import "google/api/field_behavior.proto";
import "google/api/resource.proto";
import "google/cloud/aiplatform/v1/openapi.proto";
import "google/protobuf/struct.proto";
import "google/type/latlng.proto";
option csharp_namespace = "Google.Cloud.AIPlatform.V1";
option go_package = "cloud.google.com/go/aiplatform/apiv1/aiplatformpb;aiplatformpb";
option java_multiple_files = true;
option java_outer_classname = "ToolProto";
option java_package = "com.google.cloud.aiplatform.v1";
option php_namespace = "Google\\Cloud\\AIPlatform\\V1";
option ruby_package = "Google::Cloud::AIPlatform::V1";
// Tool details that the model may use to generate response.
//
// A `Tool` is a piece of code that enables the system to interact with
// external systems to perform an action, or set of actions, outside of
// knowledge and scope of the model. A Tool object should contain exactly
// one type of Tool (e.g FunctionDeclaration, Retrieval or
// GoogleSearchRetrieval).
message Tool {
// These are available confidence level user can set to block malicious urls
// with chosen confidence and above. For understanding different confidence of
// webrisk, please refer to
// https://cloud.google.com/web-risk/docs/reference/rpc/google.cloud.webrisk.v1eap1#confidencelevel
enum PhishBlockThreshold {
// Defaults to unspecified.
PHISH_BLOCK_THRESHOLD_UNSPECIFIED = 0;
// Blocks Low and above confidence URL that is risky.
BLOCK_LOW_AND_ABOVE = 30;
// Blocks Medium and above confidence URL that is risky.
BLOCK_MEDIUM_AND_ABOVE = 40;
// Blocks High and above confidence URL that is risky.
BLOCK_HIGH_AND_ABOVE = 50;
// Blocks Higher and above confidence URL that is risky.
BLOCK_HIGHER_AND_ABOVE = 55;
// Blocks Very high and above confidence URL that is risky.
BLOCK_VERY_HIGH_AND_ABOVE = 60;
// Blocks Extremely high confidence URL that is risky.
BLOCK_ONLY_EXTREMELY_HIGH = 100;
}
// GoogleSearch tool type.
// Tool to support Google Search in Model. Powered by Google.
message GoogleSearch {
// Optional. List of domains to be excluded from the search results.
// The default limit is 2000 domains.
// Example: ["amazon.com", "facebook.com"].
repeated string exclude_domains = 3
[(google.api.field_behavior) = OPTIONAL];
// Optional. Sites with confidence level chosen & above this value will be
// blocked from the search results.
optional PhishBlockThreshold blocking_confidence = 4
[(google.api.field_behavior) = OPTIONAL];
}
// Tool that executes code generated by the model, and automatically returns
// the result to the model.
//
// See also [ExecutableCode]and [CodeExecutionResult] which are input and
// output to this tool.
message CodeExecution {}
// Tool to support computer use.
message ComputerUse {
// Represents the environment being operated, such as a web browser.
enum Environment {
// Defaults to browser.
ENVIRONMENT_UNSPECIFIED = 0;
// Operates in a web browser.
ENVIRONMENT_BROWSER = 1;
}
// Required. The environment being operated.
Environment environment = 1 [(google.api.field_behavior) = REQUIRED];
// Optional. By default, [predefined
// functions](https://cloud.google.com/vertex-ai/generative-ai/docs/computer-use#supported-actions)
// are included in the final model call. Some of them can be explicitly
// excluded from being automatically included. This can serve two purposes:
// 1. Using a more restricted / different action space.
// 2. Improving the definitions / instructions of predefined functions.
repeated string excluded_predefined_functions = 2
[(google.api.field_behavior) = OPTIONAL];
}
// Optional. Function tool type.
// One or more function declarations to be passed to the model along with the
// current user query. Model may decide to call a subset of these functions
// by populating [FunctionCall][google.cloud.aiplatform.v1.Part.function_call]
// in the response. User should provide a
// [FunctionResponse][google.cloud.aiplatform.v1.Part.function_response] for
// each function call in the next turn. Based on the function responses, Model
// will generate the final response back to the user. Maximum 128 function
// declarations can be provided.
repeated FunctionDeclaration function_declarations = 1
[(google.api.field_behavior) = OPTIONAL];
// Optional. Retrieval tool type.
// System will always execute the provided retrieval tool(s) to get external
// knowledge to answer the prompt. Retrieval results are presented to the
// model for generation.
Retrieval retrieval = 2 [(google.api.field_behavior) = OPTIONAL];
// Optional. GoogleSearch tool type.
// Tool to support Google Search in Model. Powered by Google.
GoogleSearch google_search = 7 [(google.api.field_behavior) = OPTIONAL];
// Optional. GoogleSearchRetrieval tool type.
// Specialized retrieval tool that is powered by Google search.
GoogleSearchRetrieval google_search_retrieval = 3
[(google.api.field_behavior) = OPTIONAL];
// Optional. GoogleMaps tool type.
// Tool to support Google Maps in Model.
GoogleMaps google_maps = 5 [(google.api.field_behavior) = OPTIONAL];
// Optional. Tool to support searching public web data, powered by Vertex AI
// Search and Sec4 compliance.
EnterpriseWebSearch enterprise_web_search = 6
[(google.api.field_behavior) = OPTIONAL];
// Optional. CodeExecution tool type.
// Enables the model to execute code as part of generation.
CodeExecution code_execution = 4 [(google.api.field_behavior) = OPTIONAL];
reserved 8, 9;
// Optional. Tool to support URL context retrieval.
UrlContext url_context = 10 [(google.api.field_behavior) = OPTIONAL];
// Optional. Tool to support the model interacting directly with the computer.
// If enabled, it automatically populates computer-use specific Function
// Declarations.
ComputerUse computer_use = 11 [(google.api.field_behavior) = OPTIONAL];
}
// Tool to support URL context.
message UrlContext {}
// Structured representation of a function declaration as defined by the
// [OpenAPI 3.0 specification](https://spec.openapis.org/oas/v3.0.3). Included
// in this declaration are the function name, description, parameters and
// response type. This FunctionDeclaration is a representation of a block of
// code that can be used as a `Tool` by the model and executed by the client.
message FunctionDeclaration {
// Required. The name of the function to call.
// Must start with a letter or an underscore.
// Must be a-z, A-Z, 0-9, or contain underscores, dots and dashes, with a
// maximum length of 64.
string name = 1 [(google.api.field_behavior) = REQUIRED];
// Optional. Description and purpose of the function.
// Model uses it to decide how and whether to call the function.
string description = 2 [(google.api.field_behavior) = OPTIONAL];
// Optional. Describes the parameters to this function in JSON Schema Object
// format. Reflects the Open API 3.03 Parameter Object. string Key: the name
// of the parameter. Parameter names are case sensitive. Schema Value: the
// Schema defining the type used for the parameter. For function with no
// parameters, this can be left unset. Parameter names must start with a
// letter or an underscore and must only contain chars a-z, A-Z, 0-9, or
// underscores with a maximum length of 64. Example with 1 required and 1
// optional parameter: type: OBJECT properties:
// param1:
// type: STRING
// param2:
// type: INTEGER
// required:
// - param1
Schema parameters = 3 [(google.api.field_behavior) = OPTIONAL];
// Optional. Describes the parameters to the function in JSON Schema format.
// The schema must describe an object where the properties are the parameters
// to the function. For example:
//
// ```
// {
// "type": "object",
// "properties": {
// "name": { "type": "string" },
// "age": { "type": "integer" }
// },
// "additionalProperties": false,
// "required": ["name", "age"],
// "propertyOrdering": ["name", "age"]
// }
// ```
//
// This field is mutually exclusive with `parameters`.
google.protobuf.Value parameters_json_schema = 5
[(google.api.field_behavior) = OPTIONAL];
// Optional. Describes the output from this function in JSON Schema format.
// Reflects the Open API 3.03 Response Object. The Schema defines the type
// used for the response value of the function.
Schema response = 4 [(google.api.field_behavior) = OPTIONAL];
// Optional. Describes the output from this function in JSON Schema format.
// The value specified by the schema is the response value of the function.
//
// This field is mutually exclusive with `response`.
google.protobuf.Value response_json_schema = 6
[(google.api.field_behavior) = OPTIONAL];
}
// A predicted [FunctionCall] returned from the model that contains a string
// representing the [FunctionDeclaration.name] and a structured JSON object
// containing the parameters and their values.
message FunctionCall {
// Optional. The name of the function to call.
// Matches [FunctionDeclaration.name].
string name = 1 [(google.api.field_behavior) = OPTIONAL];
// Optional. The function parameters and values in JSON object format.
// See [FunctionDeclaration.parameters] for parameter details.
google.protobuf.Struct args = 2 [(google.api.field_behavior) = OPTIONAL];
// Optional. The partial argument value of the function call.
// If provided, represents the arguments/fields that are streamed
// incrementally.
repeated PartialArg partial_args = 4 [(google.api.field_behavior) = OPTIONAL];
// Optional. Whether this is the last part of the FunctionCall.
// If true, another partial message for the current FunctionCall is expected
// to follow.
bool will_continue = 5 [(google.api.field_behavior) = OPTIONAL];
}
// Partial argument value of the function call.
message PartialArg {
// The delta of field value being streamed.
oneof delta {
// Optional. Represents a null value.
google.protobuf.NullValue null_value = 2
[(google.api.field_behavior) = OPTIONAL];
// Optional. Represents a double value.
double number_value = 3 [(google.api.field_behavior) = OPTIONAL];
// Optional. Represents a string value.
string string_value = 4 [(google.api.field_behavior) = OPTIONAL];
// Optional. Represents a boolean value.
bool bool_value = 5 [(google.api.field_behavior) = OPTIONAL];
}
// Required. A JSON Path (RFC 9535) to the argument being streamed.
// https://datatracker.ietf.org/doc/html/rfc9535. e.g. "$.foo.bar[0].data".
string json_path = 1 [(google.api.field_behavior) = REQUIRED];
// Optional. Whether this is not the last part of the same json_path.
// If true, another PartialArg message for the current json_path is expected
// to follow.
bool will_continue = 6 [(google.api.field_behavior) = OPTIONAL];
}
// A datatype containing media that is part of a `FunctionResponse` message.
//
// A `FunctionResponsePart` consists of data which has an associated datatype. A
// `FunctionResponsePart` can only contain one of the accepted types in
// `FunctionResponsePart.data`.
//
// A `FunctionResponsePart` must have a fixed IANA MIME type identifying the
// type and subtype of the media if the `inline_data` field is filled with raw
// bytes.
message FunctionResponsePart {
// The data of the function response part.
oneof data {
// Inline media bytes.
FunctionResponseBlob inline_data = 1;
// URI based data.
FunctionResponseFileData file_data = 2;
}
}
// Raw media bytes for function response.
//
// Text should not be sent as raw bytes, use the 'text' field.
message FunctionResponseBlob {
// Required. The IANA standard MIME type of the source data.
string mime_type = 1 [(google.api.field_behavior) = REQUIRED];
// Required. Raw bytes.
bytes data = 2 [(google.api.field_behavior) = REQUIRED];
// Optional. Display name of the blob.
//
// Used to provide a label or filename to distinguish blobs.
//
// This field is only returned in PromptMessage for prompt management.
// It is currently used in the Gemini GenerateContent calls only when server
// side tools (code_execution, google_search, and url_context) are enabled.
string display_name = 4 [(google.api.field_behavior) = OPTIONAL];
}
// URI based data for function response.
message FunctionResponseFileData {
// Required. The IANA standard MIME type of the source data.
string mime_type = 1 [(google.api.field_behavior) = REQUIRED];
// Required. URI.
string file_uri = 2 [(google.api.field_behavior) = REQUIRED];
// Optional. Display name of the file data.
//
// Used to provide a label or filename to distinguish file datas.
//
// This field is only returned in PromptMessage for prompt management.
// It is currently used in the Gemini GenerateContent calls only when server
// side tools (code_execution, google_search, and url_context) are enabled.
string display_name = 3 [(google.api.field_behavior) = OPTIONAL];
}
// The result output from a [FunctionCall] that contains a string representing
// the [FunctionDeclaration.name] and a structured JSON object containing any
// output from the function is used as context to the model. This should contain
// the result of a [FunctionCall] made based on model prediction.
message FunctionResponse {
// Required. The name of the function to call.
// Matches [FunctionDeclaration.name] and [FunctionCall.name].
string name = 1 [(google.api.field_behavior) = REQUIRED];
// Required. The function response in JSON object format.
// Use "output" key to specify function output and "error" key to specify
// error details (if any). If "output" and "error" keys are not specified,
// then whole "response" is treated as function output.
google.protobuf.Struct response = 2 [(google.api.field_behavior) = REQUIRED];
// Optional. Ordered `Parts` that constitute a function response. Parts may
// have different IANA MIME types.
repeated FunctionResponsePart parts = 4
[(google.api.field_behavior) = OPTIONAL];
}
// Code generated by the model that is meant to be executed, and the result
// returned to the model.
//
// Generated when using the [FunctionDeclaration] tool and
// [FunctionCallingConfig] mode is set to [Mode.CODE].
message ExecutableCode {
// Supported programming languages for the generated code.
enum Language {
// Unspecified language. This value should not be used.
LANGUAGE_UNSPECIFIED = 0;
// Python >= 3.10, with numpy and simpy available.
PYTHON = 1;
}
// Required. Programming language of the `code`.
Language language = 1 [(google.api.field_behavior) = REQUIRED];
// Required. The code to be executed.
string code = 2 [(google.api.field_behavior) = REQUIRED];
}
// Result of executing the [ExecutableCode].
//
// Always follows a `part` containing the [ExecutableCode].
message CodeExecutionResult {
// Enumeration of possible outcomes of the code execution.
enum Outcome {
// Unspecified status. This value should not be used.
OUTCOME_UNSPECIFIED = 0;
// Code execution completed successfully.
OUTCOME_OK = 1;
// Code execution finished but with a failure. `stderr` should contain the
// reason.
OUTCOME_FAILED = 2;
// Code execution ran for too long, and was cancelled. There may or may not
// be a partial output present.
OUTCOME_DEADLINE_EXCEEDED = 3;
}
// Required. Outcome of the code execution.
Outcome outcome = 1 [(google.api.field_behavior) = REQUIRED];
// Optional. Contains stdout when code execution is successful, stderr or
// other description otherwise.
string output = 2 [(google.api.field_behavior) = OPTIONAL];
}
// Defines a retrieval tool that model can call to access external knowledge.
message Retrieval {
// The source of the retrieval.
oneof source {
// Set to use data source powered by Vertex AI Search.
VertexAISearch vertex_ai_search = 2;
// Set to use data source powered by Vertex RAG store.
// User data is uploaded via the VertexRagDataService.
VertexRagStore vertex_rag_store = 4;
}
// Optional. Deprecated. This option is no longer supported.
bool disable_attribution = 3
[deprecated = true, (google.api.field_behavior) = OPTIONAL];
}
// Retrieve from Vertex RAG Store for grounding.
message VertexRagStore {
// The definition of the Rag resource.
message RagResource {
// Optional. RagCorpora resource name.
// Format:
// `projects/{project}/locations/{location}/ragCorpora/{rag_corpus}`
string rag_corpus = 1 [
(google.api.field_behavior) = OPTIONAL,
(google.api.resource_reference) = {
type: "aiplatform.googleapis.com/RagCorpus"
}
];
// Optional. rag_file_id. The files should be in the same rag_corpus set in
// rag_corpus field.
repeated string rag_file_ids = 2 [(google.api.field_behavior) = OPTIONAL];
}
// Optional. The representation of the rag source. It can be used to specify
// corpus only or ragfiles. Currently only support one corpus or multiple
// files from one corpus. In the future we may open up multiple corpora
// support.
repeated RagResource rag_resources = 4
[(google.api.field_behavior) = OPTIONAL];
// Optional. Number of top k results to return from the selected corpora.
optional int32 similarity_top_k = 2
[deprecated = true, (google.api.field_behavior) = OPTIONAL];
// Optional. Only return results with vector distance smaller than the
// threshold.
optional double vector_distance_threshold = 3
[deprecated = true, (google.api.field_behavior) = OPTIONAL];
// Optional. The retrieval config for the Rag query.
RagRetrievalConfig rag_retrieval_config = 6
[(google.api.field_behavior) = OPTIONAL];
}
// Retrieve from Vertex AI Search datastore or engine for grounding.
// datastore and engine are mutually exclusive.
// See https://cloud.google.com/products/agent-builder
message VertexAISearch {
// Define data stores within engine to filter on in a search call and
// configurations for those data stores. For more information, see
// https://cloud.google.com/generative-ai-app-builder/docs/reference/rpc/google.cloud.discoveryengine.v1#datastorespec
message DataStoreSpec {
// Full resource name of DataStore, such as
// Format:
// `projects/{project}/locations/{location}/collections/{collection}/dataStores/{dataStore}`
string data_store = 1;
// Optional. Filter specification to filter documents in the data store
// specified by data_store field. For more information on filtering, see
// [Filtering](https://cloud.google.com/generative-ai-app-builder/docs/filter-search-metadata)
string filter = 2 [(google.api.field_behavior) = OPTIONAL];
}
// Optional. Fully-qualified Vertex AI Search data store resource ID.
// Format:
// `projects/{project}/locations/{location}/collections/{collection}/dataStores/{dataStore}`
string datastore = 1 [(google.api.field_behavior) = OPTIONAL];
// Optional. Fully-qualified Vertex AI Search engine resource ID.
// Format:
// `projects/{project}/locations/{location}/collections/{collection}/engines/{engine}`
string engine = 2 [(google.api.field_behavior) = OPTIONAL];
// Optional. Number of search results to return per query.
// The default value is 10.
// The maximumm allowed value is 10.
int32 max_results = 3 [(google.api.field_behavior) = OPTIONAL];
// Optional. Filter strings to be passed to the search API.
string filter = 4 [(google.api.field_behavior) = OPTIONAL];
// Specifications that define the specific DataStores to be searched, along
// with configurations for those data stores. This is only considered for
// Engines with multiple data stores.
// It should only be set if engine is used.
repeated DataStoreSpec data_store_specs = 5;
}
// Tool to retrieve public web data for grounding, powered by Google.
message GoogleSearchRetrieval {
// Specifies the dynamic retrieval configuration for the given source.
DynamicRetrievalConfig dynamic_retrieval_config = 2;
}
// Tool to retrieve public maps data for grounding, powered by Google.
message GoogleMaps {
// If true, include the widget context token in the response.
bool enable_widget = 1 [(google.api.field_behavior) = OPTIONAL];
}
// Tool to search public web data, powered by Vertex AI Search and Sec4
// compliance.
message EnterpriseWebSearch {
// Optional. List of domains to be excluded from the search results.
// The default limit is 2000 domains.
repeated string exclude_domains = 1 [(google.api.field_behavior) = OPTIONAL];
// Optional. Sites with confidence level chosen & above this value will be
// blocked from the search results.
optional Tool.PhishBlockThreshold blocking_confidence = 2
[(google.api.field_behavior) = OPTIONAL];
}
// Describes the options to customize dynamic retrieval.
message DynamicRetrievalConfig {
// The mode of the predictor to be used in dynamic retrieval.
enum Mode {
// Always trigger retrieval.
MODE_UNSPECIFIED = 0;
// Run retrieval only when system decides it is necessary.
MODE_DYNAMIC = 1;
}
// The mode of the predictor to be used in dynamic retrieval.
Mode mode = 1;
// Optional. The threshold to be used in dynamic retrieval.
// If not set, a system default value is used.
optional float dynamic_threshold = 2 [(google.api.field_behavior) = OPTIONAL];
}
// Tool config. This config is shared for all tools provided in the request.
message ToolConfig {
// Optional. Function calling config.
FunctionCallingConfig function_calling_config = 1
[(google.api.field_behavior) = OPTIONAL];
// Optional. Retrieval config.
RetrievalConfig retrieval_config = 2 [(google.api.field_behavior) = OPTIONAL];
}
// Function calling config.
message FunctionCallingConfig {
// Function calling mode.
enum Mode {
reserved 4;
// Unspecified function calling mode. This value should not be used.
MODE_UNSPECIFIED = 0;
// Default model behavior, model decides to predict either function calls
// or natural language response.
AUTO = 1;
// Model is constrained to always predicting function calls only.
// If "allowed_function_names" are set, the predicted function calls will be
// limited to any one of "allowed_function_names", else the predicted
// function calls will be any one of the provided "function_declarations".
ANY = 2;
// Model will not predict any function calls. Model behavior is same as when
// not passing any function declarations.
NONE = 3;
// Model is constrained to predict either function calls or natural language
// response.
// If [allowed_function_names][FunctionCallingConfig.allowed_function_names]
// are set, the predicted function calls will be limited to any one of
// `allowed_function_names`, else the predicted function calls will be any
// one of the provided [FunctionDeclaration].
VALIDATED = 5;
}
// Optional. Function calling mode.
Mode mode = 1 [(google.api.field_behavior) = OPTIONAL];
// Optional. Function names to call. Only set when the Mode is ANY. Function
// names should match [FunctionDeclaration.name]. With mode set to ANY, model
// will predict a function call from the set of function names provided.
repeated string allowed_function_names = 2
[(google.api.field_behavior) = OPTIONAL];
// Optional. When set to true, arguments of a single function call will be
// streamed out in multiple parts/contents/responses. Partial parameter
// results will be returned in the [FunctionCall.partial_args] field.
bool stream_function_call_arguments = 4
[(google.api.field_behavior) = OPTIONAL];
}
// Retrieval config.
message RetrievalConfig {
// The location of the user.
optional google.type.LatLng lat_lng = 1;
// The language code of the user.
optional string language_code = 2;
}
// Specifies the context retrieval config.
message RagRetrievalConfig {
// Config for filters.
message Filter {
// Filter contexts retrieved from the vector DB based on either vector
// distance or vector similarity.
oneof vector_db_threshold {
// Optional. Only returns contexts with vector distance smaller than the
// threshold.
double vector_distance_threshold = 3
[(google.api.field_behavior) = OPTIONAL];
// Optional. Only returns contexts with vector similarity larger than the
// threshold.
double vector_similarity_threshold = 4
[(google.api.field_behavior) = OPTIONAL];
}
// Optional. String for metadata filtering.
string metadata_filter = 2 [(google.api.field_behavior) = OPTIONAL];
}
// Config for ranking and reranking.
message Ranking {
// Config for Rank Service.
message RankService {
// Optional. The model name of the rank service.
// Format: `semantic-ranker-512@latest`
optional string model_name = 1 [(google.api.field_behavior) = OPTIONAL];
}
// Config for LlmRanker.
message LlmRanker {
// Optional. The model name used for ranking.
// Format: `gemini-1.5-pro`
optional string model_name = 1 [(google.api.field_behavior) = OPTIONAL];
}
// Config options for ranking. Currently only Rank Service is supported.
oneof ranking_config {
// Optional. Config for Rank Service.
RankService rank_service = 1 [(google.api.field_behavior) = OPTIONAL];
// Optional. Config for LlmRanker.
LlmRanker llm_ranker = 3 [(google.api.field_behavior) = OPTIONAL];
}
}
// Optional. The number of contexts to retrieve.
int32 top_k = 1 [(google.api.field_behavior) = OPTIONAL];
// Optional. Config for filters.
Filter filter = 3 [(google.api.field_behavior) = OPTIONAL];
// Optional. Config for ranking and reranking.
Ranking ranking = 4 [(google.api.field_behavior) = OPTIONAL];
}