From 731d7f2ab6e4e2ea15030c95039e2cb66174d4fb Mon Sep 17 00:00:00 2001 From: Google APIs Date: Fri, 22 Aug 2025 10:12:28 -0700 Subject: [PATCH] feat: add gkerecommender GkeInferenceQuickstart API v1 PiperOrigin-RevId: 798253849 --- google/cloud/gkerecommender/v1/BUILD.bazel | 380 +++++++++++ .../gkerecommender/v1/gkerecommender.proto | 589 ++++++++++++++++++ .../gkerecommender_grpc_service_config.json | 15 + .../gkerecommender/v1/gkerecommender_v1.yaml | 17 + 4 files changed, 1001 insertions(+) create mode 100644 google/cloud/gkerecommender/v1/BUILD.bazel create mode 100644 google/cloud/gkerecommender/v1/gkerecommender.proto create mode 100644 google/cloud/gkerecommender/v1/gkerecommender_grpc_service_config.json create mode 100644 google/cloud/gkerecommender/v1/gkerecommender_v1.yaml diff --git a/google/cloud/gkerecommender/v1/BUILD.bazel b/google/cloud/gkerecommender/v1/BUILD.bazel new file mode 100644 index 0000000000..8115022bce --- /dev/null +++ b/google/cloud/gkerecommender/v1/BUILD.bazel @@ -0,0 +1,380 @@ +# This file was automatically generated by BuildFileGenerator +# https://github.com/googleapis/rules_gapic/tree/master/bazel + +# Most of the manual changes to this file will be overwritten. +# It's **only** allowed to change the following rule attribute values: +# - names of *_gapic_assembly_* rules +# - certain parameters of *_gapic_library rules, including but not limited to: +# * extra_protoc_parameters +# * extra_protoc_file_parameters +# The complete list of preserved parameters can be found in the source code. + +# buildifier: disable=load-on-top + +# This is an API workspace, having public visibility by default makes perfect sense. +package(default_visibility = ["//visibility:public"]) + +############################################################################## +# Common +############################################################################## +# buildifier: disable=same-origin-load +load("@com_google_googleapis_imports//:imports.bzl", "proto_library_with_info") +load("@rules_proto//proto:defs.bzl", "proto_library") + +proto_library( + name = "gkerecommender_proto", + srcs = [ + "gkerecommender.proto", + ], + deps = [ + "//google/api:annotations_proto", + "//google/api:client_proto", + "//google/api:field_behavior_proto", + ], +) + +proto_library_with_info( + name = "gkerecommender_proto_with_info", + deps = [ + ":gkerecommender_proto", + "//google/cloud:common_resources_proto", + ], +) + +############################################################################## +# Java +############################################################################## +# buildifier: disable=same-origin-load +load( + "@com_google_googleapis_imports//:imports.bzl", + "java_gapic_assembly_gradle_pkg", + "java_gapic_library", + "java_gapic_test", + "java_grpc_library", + "java_proto_library", +) + +java_proto_library( + name = "gkerecommender_java_proto", + deps = [":gkerecommender_proto"], +) + +java_grpc_library( + name = "gkerecommender_java_grpc", + srcs = [":gkerecommender_proto"], + deps = [":gkerecommender_java_proto"], +) + +java_gapic_library( + name = "gkerecommender_java_gapic", + srcs = [":gkerecommender_proto_with_info"], + gapic_yaml = None, + grpc_service_config = "gkerecommender_grpc_service_config.json", + rest_numeric_enums = True, + service_yaml = "gkerecommender_v1.yaml", + test_deps = [ + ":gkerecommender_java_grpc", + ], + transport = "grpc+rest", + deps = [ + ":gkerecommender_java_proto", + "//google/api:api_java_proto", + ], +) + +java_gapic_test( + name = "gkerecommender_java_gapic_test_suite", + test_classes = [ + "com.google.cloud.gkerecommender.v1.GkeInferenceQuickstartClientHttpJsonTest", + "com.google.cloud.gkerecommender.v1.GkeInferenceQuickstartClientTest", + ], + runtime_deps = [":gkerecommender_java_gapic_test"], +) + +# Open Source Packages +java_gapic_assembly_gradle_pkg( + name = "google-cloud-gkerecommender-v1-java", + include_samples = True, + transport = "grpc+rest", + deps = [ + ":gkerecommender_java_gapic", + ":gkerecommender_java_grpc", + ":gkerecommender_java_proto", + ":gkerecommender_proto", + ], +) + +############################################################################## +# Go +############################################################################## +# buildifier: disable=same-origin-load +load( + "@com_google_googleapis_imports//:imports.bzl", + "go_gapic_assembly_pkg", + "go_gapic_library", + "go_proto_library", +) + +go_proto_library( + name = "gkerecommender_go_proto", + compilers = ["@io_bazel_rules_go//proto:go_grpc"], + importpath = "cloud.google.com/go/gkerecommender/apiv1/gkerecommenderpb", + protos = [":gkerecommender_proto"], + deps = [ + "//google/api:annotations_go_proto", + ], +) + +go_gapic_library( + name = "gkerecommender_go_gapic", + srcs = [":gkerecommender_proto_with_info"], + grpc_service_config = "gkerecommender_grpc_service_config.json", + importpath = "cloud.google.com/go/gkerecommender/apiv1;gkerecommender", + metadata = True, + release_level = "beta", + rest_numeric_enums = True, + service_yaml = "gkerecommender_v1.yaml", + transport = "grpc+rest", + deps = [ + ":gkerecommender_go_proto", + ], +) + +# Open Source Packages +go_gapic_assembly_pkg( + name = "gapi-cloud-gkerecommender-v1-go", + deps = [ + ":gkerecommender_go_gapic", + ":gkerecommender_go_gapic_srcjar-metadata.srcjar", + ":gkerecommender_go_gapic_srcjar-snippets.srcjar", + ":gkerecommender_go_gapic_srcjar-test.srcjar", + ":gkerecommender_go_proto", + ], +) + +############################################################################## +# Python +############################################################################## +# buildifier: disable=same-origin-load +load( + "@com_google_googleapis_imports//:imports.bzl", + "py_gapic_assembly_pkg", + "py_gapic_library", + "py_test", +) + +py_gapic_library( + name = "gkerecommender_py_gapic", + srcs = [":gkerecommender_proto"], + grpc_service_config = "gkerecommender_grpc_service_config.json", + rest_numeric_enums = True, + service_yaml = "gkerecommender_v1.yaml", + transport = "grpc+rest", + deps = [], +) + +py_test( + name = "gkerecommender_py_gapic_test", + srcs = [ + "gkerecommender_py_gapic_pytest.py", + "gkerecommender_py_gapic_test.py", + ], + legacy_create_init = False, + deps = [":gkerecommender_py_gapic"], +) + +# Open Source Packages +py_gapic_assembly_pkg( + name = "gkerecommender-v1-py", + deps = [ + ":gkerecommender_py_gapic", + ], +) + +############################################################################## +# PHP +############################################################################## +# buildifier: disable=same-origin-load +load( + "@com_google_googleapis_imports//:imports.bzl", + "php_gapic_assembly_pkg", + "php_gapic_library", + "php_proto_library", +) + +php_proto_library( + name = "gkerecommender_php_proto", + deps = [":gkerecommender_proto"], +) + +php_gapic_library( + name = "gkerecommender_php_gapic", + srcs = [":gkerecommender_proto_with_info"], + grpc_service_config = "gkerecommender_grpc_service_config.json", + migration_mode = "NEW_SURFACE_ONLY", + rest_numeric_enums = True, + service_yaml = "gkerecommender_v1.yaml", + transport = "grpc+rest", + deps = [ + ":gkerecommender_php_proto", + ], +) + +# Open Source Packages +php_gapic_assembly_pkg( + name = "google-cloud-gkerecommender-v1-php", + deps = [ + ":gkerecommender_php_gapic", + ":gkerecommender_php_proto", + ], +) + +############################################################################## +# Node.js +############################################################################## +# buildifier: disable=same-origin-load +load( + "@com_google_googleapis_imports//:imports.bzl", + "nodejs_gapic_assembly_pkg", + "nodejs_gapic_library", +) + +nodejs_gapic_library( + name = "gkerecommender_nodejs_gapic", + package_name = "@google-cloud/gkerecommender", + src = ":gkerecommender_proto_with_info", + extra_protoc_parameters = ["metadata"], + grpc_service_config = "gkerecommender_grpc_service_config.json", + package = "google.cloud.gkerecommender.v1", + rest_numeric_enums = True, + service_yaml = "gkerecommender_v1.yaml", + transport = "grpc+rest", + deps = [], +) + +nodejs_gapic_assembly_pkg( + name = "gkerecommender-v1-nodejs", + deps = [ + ":gkerecommender_nodejs_gapic", + ":gkerecommender_proto", + ], +) + +############################################################################## +# Ruby +############################################################################## +# buildifier: disable=same-origin-load +load( + "@com_google_googleapis_imports//:imports.bzl", + "ruby_cloud_gapic_library", + "ruby_gapic_assembly_pkg", + "ruby_grpc_library", + "ruby_proto_library", +) + +ruby_proto_library( + name = "gkerecommender_ruby_proto", + deps = [":gkerecommender_proto"], +) + +ruby_grpc_library( + name = "gkerecommender_ruby_grpc", + srcs = [":gkerecommender_proto"], + deps = [":gkerecommender_ruby_proto"], +) + +ruby_cloud_gapic_library( + name = "gkerecommender_ruby_gapic", + srcs = [":gkerecommender_proto_with_info"], + extra_protoc_parameters = [ + "ruby-cloud-gem-name=google-cloud-gkerecommender-v1", + ], + grpc_service_config = "gkerecommender_grpc_service_config.json", + rest_numeric_enums = True, + service_yaml = "gkerecommender_v1.yaml", + transport = "grpc+rest", + deps = [ + ":gkerecommender_ruby_grpc", + ":gkerecommender_ruby_proto", + ], +) + +# Open Source Packages +ruby_gapic_assembly_pkg( + name = "google-cloud-gkerecommender-v1-ruby", + deps = [ + ":gkerecommender_ruby_gapic", + ":gkerecommender_ruby_grpc", + ":gkerecommender_ruby_proto", + ], +) + +############################################################################## +# C# +############################################################################## +# buildifier: disable=same-origin-load +load( + "@com_google_googleapis_imports//:imports.bzl", + "csharp_gapic_assembly_pkg", + "csharp_gapic_library", + "csharp_grpc_library", + "csharp_proto_library", +) + +csharp_proto_library( + name = "gkerecommender_csharp_proto", + extra_opts = [], + deps = [":gkerecommender_proto"], +) + +csharp_grpc_library( + name = "gkerecommender_csharp_grpc", + srcs = [":gkerecommender_proto"], + deps = [":gkerecommender_csharp_proto"], +) + +csharp_gapic_library( + name = "gkerecommender_csharp_gapic", + srcs = [":gkerecommender_proto_with_info"], + common_resources_config = "@gax_dotnet//:Google.Api.Gax/ResourceNames/CommonResourcesConfig.json", + grpc_service_config = "gkerecommender_grpc_service_config.json", + rest_numeric_enums = True, + service_yaml = "gkerecommender_v1.yaml", + transport = "grpc+rest", + deps = [ + ":gkerecommender_csharp_grpc", + ":gkerecommender_csharp_proto", + ], +) + +# Open Source Packages +csharp_gapic_assembly_pkg( + name = "google-cloud-gkerecommender-v1-csharp", + deps = [ + ":gkerecommender_csharp_gapic", + ":gkerecommender_csharp_grpc", + ":gkerecommender_csharp_proto", + ], +) + +############################################################################## +# C++ +############################################################################## +# buildifier: disable=same-origin-load +load( + "@com_google_googleapis_imports//:imports.bzl", + "cc_grpc_library", + "cc_proto_library", +) + +cc_proto_library( + name = "gkerecommender_cc_proto", + deps = [":gkerecommender_proto"], +) + +cc_grpc_library( + name = "gkerecommender_cc_grpc", + srcs = [":gkerecommender_proto"], + grpc_only = True, + deps = [":gkerecommender_cc_proto"], +) diff --git a/google/cloud/gkerecommender/v1/gkerecommender.proto b/google/cloud/gkerecommender/v1/gkerecommender.proto new file mode 100644 index 0000000000..af9c31768d --- /dev/null +++ b/google/cloud/gkerecommender/v1/gkerecommender.proto @@ -0,0 +1,589 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.cloud.gkerecommender.v1; + +import "google/api/annotations.proto"; +import "google/api/client.proto"; +import "google/api/field_behavior.proto"; + +option csharp_namespace = "Google.Cloud.GkeRecommender.V1"; +option go_package = "cloud.google.com/go/gkerecommender/apiv1/gkerecommenderpb;gkerecommenderpb"; +option java_multiple_files = true; +option java_outer_classname = "GkeRecommenderProto"; +option java_package = "com.google.cloud.gkerecommender.v1"; +option php_namespace = "Google\\Cloud\\GkeRecommender\\V1"; +option ruby_package = "Google::Cloud::GkeRecommender::V1"; + +// GKE Inference Quickstart (GIQ) service provides profiles with performance +// metrics for popular models and model servers across multiple accelerators. +// These profiles help generate optimized best practices for running inference +// on GKE. +service GkeInferenceQuickstart { + option (google.api.default_host) = "gkerecommender.googleapis.com"; + option (google.api.oauth_scopes) = + "https://www.googleapis.com/auth/cloud-platform"; + + // Fetches available models. Open-source models follow the Huggingface Hub + // `owner/model_name` format. + rpc FetchModels(FetchModelsRequest) returns (FetchModelsResponse) { + option (google.api.http) = { + get: "/v1/models:fetch" + }; + } + + // Fetches available model servers. Open-source model servers use simplified, + // lowercase names (e.g., `vllm`). + rpc FetchModelServers(FetchModelServersRequest) + returns (FetchModelServersResponse) { + option (google.api.http) = { + get: "/v1/modelServers:fetch" + }; + } + + // Fetches available model server versions. Open-source servers use their own + // versioning schemas (e.g., `vllm` uses semver like `v1.0.0`). + // + // Some model servers have different versioning schemas depending on the + // accelerator. For example, `vllm` uses semver on GPUs, but returns nightly + // build tags on TPUs. All available versions will be returned when different + // schemas are present. + rpc FetchModelServerVersions(FetchModelServerVersionsRequest) + returns (FetchModelServerVersionsResponse) { + option (google.api.http) = { + get: "/v1/modelServerVersions:fetch" + }; + } + + // Fetches available profiles. A profile contains performance metrics and + // cost information for a specific model server setup. Profiles can be + // filtered by parameters. If no filters are provided, all profiles are + // returned. + // + // Profiles display a single value per performance metric based on the + // provided performance requirements. If no requirements are given, the + // metrics represent the inflection point. See [Run best practice inference + // with GKE Inference Quickstart + // recipes](https://cloud.google.com/kubernetes-engine/docs/how-to/machine-learning/inference/inference-quickstart#how) + // for details. + rpc FetchProfiles(FetchProfilesRequest) returns (FetchProfilesResponse) { + option (google.api.http) = { + post: "/v1/profiles:fetch" + body: "*" + }; + } + + // Generates an optimized deployment manifest for a given model and model + // server, based on the specified accelerator, performance targets, and + // configurations. See [Run best practice inference with GKE Inference + // Quickstart + // recipes](https://cloud.google.com/kubernetes-engine/docs/how-to/machine-learning/inference/inference-quickstart) + // for deployment details. + rpc GenerateOptimizedManifest(GenerateOptimizedManifestRequest) + returns (GenerateOptimizedManifestResponse) { + option (google.api.http) = { + post: "/v1/optimizedManifest:generate" + body: "*" + }; + } + + // Fetches all of the benchmarking data available for a profile. Benchmarking + // data returns all of the performance metrics available for a given model + // server setup on a given instance type. + rpc FetchBenchmarkingData(FetchBenchmarkingDataRequest) + returns (FetchBenchmarkingDataResponse) { + option (google.api.http) = { + post: "/v1/benchmarkingData:fetch" + body: "*" + }; + } +} + +// Request message for +// [GkeInferenceQuickstart.FetchModels][google.cloud.gkerecommender.v1.GkeInferenceQuickstart.FetchModels]. +message FetchModelsRequest { + // Optional. The target number of results to return in a single response. + // If not specified, a default value will be chosen by the service. + // Note that the response may include a partial list and a caller should + // only rely on the response's + // [next_page_token][google.cloud.gkerecommender.v1.FetchModelsResponse.next_page_token] + // to determine if there are more instances left to be queried. + optional int32 page_size = 1 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The value of + // [next_page_token][google.cloud.gkerecommender.v1.FetchModelsResponse.next_page_token] + // received from a previous `FetchModelsRequest` call. + // Provide this to retrieve the subsequent page in a multi-page list of + // results. When paginating, all other parameters provided to + // `FetchModelsRequest` must match the call that provided the page token. + optional string page_token = 2 [(google.api.field_behavior) = OPTIONAL]; +} + +// Response message for +// [GkeInferenceQuickstart.FetchModels][google.cloud.gkerecommender.v1.GkeInferenceQuickstart.FetchModels]. +message FetchModelsResponse { + // Output only. List of available models. Open-source models follow the + // Huggingface Hub `owner/model_name` format. + repeated string models = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. A token which may be sent as + // [page_token][FetchModelsResponse.page_token] in a subsequent + // `FetchModelsResponse` call to retrieve the next page of results. + // If this field is omitted or empty, then there are no more results to + // return. + string next_page_token = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// Request message for +// [GkeInferenceQuickstart.FetchModelServers][google.cloud.gkerecommender.v1.GkeInferenceQuickstart.FetchModelServers]. +message FetchModelServersRequest { + // Required. The model for which to list model servers. Open-source models + // follow the Huggingface Hub `owner/model_name` format. Use + // [GkeInferenceQuickstart.FetchModels][google.cloud.gkerecommender.v1.GkeInferenceQuickstart.FetchModels] + // to find available models. + string model = 1 [(google.api.field_behavior) = REQUIRED]; + + // Optional. The target number of results to return in a single response. + // If not specified, a default value will be chosen by the service. + // Note that the response may include a partial list and a caller should + // only rely on the response's + // [next_page_token][google.cloud.gkerecommender.v1.FetchModelServersResponse.next_page_token] + // to determine if there are more instances left to be queried. + optional int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The value of + // [next_page_token][google.cloud.gkerecommender.v1.FetchModelServersResponse.next_page_token] + // received from a previous `FetchModelServersRequest` call. + // Provide this to retrieve the subsequent page in a multi-page list of + // results. When paginating, all other parameters provided to + // `FetchModelServersRequest` must match the call that provided the page + // token. + optional string page_token = 3 [(google.api.field_behavior) = OPTIONAL]; +} + +// Response message for +// [GkeInferenceQuickstart.FetchModelServers][google.cloud.gkerecommender.v1.GkeInferenceQuickstart.FetchModelServers]. +message FetchModelServersResponse { + // Output only. List of available model servers. Open-source model servers use + // simplified, lowercase names (e.g., `vllm`). + repeated string model_servers = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. A token which may be sent as + // [page_token][FetchModelServersResponse.page_token] in a subsequent + // `FetchModelServersResponse` call to retrieve the next page of results. + // If this field is omitted or empty, then there are no more results to + // return. + string next_page_token = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// Request message for +// [GkeInferenceQuickstart.FetchModelServerVersions][google.cloud.gkerecommender.v1.GkeInferenceQuickstart.FetchModelServerVersions]. +message FetchModelServerVersionsRequest { + // Required. The model for which to list model server versions. Open-source + // models follow the Huggingface Hub `owner/model_name` format. Use + // [GkeInferenceQuickstart.FetchModels][google.cloud.gkerecommender.v1.GkeInferenceQuickstart.FetchModels] + // to find available models. + string model = 1 [(google.api.field_behavior) = REQUIRED]; + + // Required. The model server for which to list versions. Open-source model + // servers use simplified, lowercase names (e.g., `vllm`). Use + // [GkeInferenceQuickstart.FetchModelServers][google.cloud.gkerecommender.v1.GkeInferenceQuickstart.FetchModelServers] + // to find available model servers. + string model_server = 2 [(google.api.field_behavior) = REQUIRED]; + + // Optional. The target number of results to return in a single response. + // If not specified, a default value will be chosen by the service. + // Note that the response may include a partial list and a caller should + // only rely on the response's + // [next_page_token][google.cloud.gkerecommender.v1.FetchModelServerVersionsResponse.next_page_token] + // to determine if there are more instances left to be queried. + optional int32 page_size = 3 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The value of + // [next_page_token][google.cloud.gkerecommender.v1.FetchModelServerVersionsResponse.next_page_token] + // received from a previous `FetchModelServerVersionsRequest` call. + // Provide this to retrieve the subsequent page in a multi-page list of + // results. When paginating, all other parameters provided to + // `FetchModelServerVersionsRequest` must match the call that provided the + // page token. + optional string page_token = 4 [(google.api.field_behavior) = OPTIONAL]; +} + +// Response message for +// [GkeInferenceQuickstart.FetchModelServerVersions][google.cloud.gkerecommender.v1.GkeInferenceQuickstart.FetchModelServerVersions]. +message FetchModelServerVersionsResponse { + // Output only. A list of available model server versions. + repeated string model_server_versions = 1 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. A token which may be sent as + // [page_token][FetchModelServerVersionsResponse.page_token] in a subsequent + // `FetchModelServerVersionsResponse` call to retrieve the next page of + // results. If this field is omitted or empty, then there are no more results + // to return. + string next_page_token = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// Request message for +// [GkeInferenceQuickstart.FetchBenchmarkingData][google.cloud.gkerecommender.v1.GkeInferenceQuickstart.FetchBenchmarkingData]. +message FetchBenchmarkingDataRequest { + // Required. The model server configuration to get benchmarking data for. Use + // [GkeInferenceQuickstart.FetchProfiles][google.cloud.gkerecommender.v1.GkeInferenceQuickstart.FetchProfiles] + // to find valid configurations. + ModelServerInfo model_server_info = 1 + [(google.api.field_behavior) = REQUIRED]; + + // Optional. The instance type to filter benchmarking data. Instance types are + // in the format `a2-highgpu-1g`. If not provided, all instance types for the + // given profile's `model_server_info` will be returned. Use + // [GkeInferenceQuickstart.FetchProfiles][google.cloud.gkerecommender.v1.GkeInferenceQuickstart.FetchProfiles] + // to find available instance types. + string instance_type = 3 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The pricing model to use for the benchmarking data. Defaults to + // `spot`. + string pricing_model = 4 [(google.api.field_behavior) = OPTIONAL]; +} + +// Response message for +// [GkeInferenceQuickstart.FetchBenchmarkingData][google.cloud.gkerecommender.v1.GkeInferenceQuickstart.FetchBenchmarkingData]. +message FetchBenchmarkingDataResponse { + // Output only. List of profiles containing their respective benchmarking + // data. + repeated Profile profile = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// Request message for +// [GkeInferenceQuickstart.FetchProfiles][google.cloud.gkerecommender.v1.GkeInferenceQuickstart.FetchProfiles]. +message FetchProfilesRequest { + // Optional. The model to filter profiles by. Open-source models follow the + // Huggingface Hub `owner/model_name` format. If not provided, all models are + // returned. Use + // [GkeInferenceQuickstart.FetchModels][google.cloud.gkerecommender.v1.GkeInferenceQuickstart.FetchModels] + // to find available models. + string model = 1 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The model server to filter profiles by. If not provided, all + // model servers are returned. Use + // [GkeInferenceQuickstart.FetchModelServers][google.cloud.gkerecommender.v1.GkeInferenceQuickstart.FetchModelServers] + // to find available model servers for a given model. + string model_server = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The model server version to filter profiles by. If not provided, + // all model server versions are returned. Use + // [GkeInferenceQuickstart.FetchModelServerVersions][google.cloud.gkerecommender.v1.GkeInferenceQuickstart.FetchModelServerVersions] + // to find available versions for a given model and server. + string model_server_version = 3 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The performance requirements to filter profiles. Profiles that do + // not meet these requirements are filtered out. If not provided, all profiles + // are returned. + PerformanceRequirements performance_requirements = 4 + [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The target number of results to return in a single response. If + // not specified, a default value will be chosen by the service. Note that the + // response may include a partial list and a caller should only rely on the + // response's + // [next_page_token][google.cloud.gkerecommender.v1.FetchProfilesResponse.next_page_token] + // to determine if there are more instances left to be queried. + optional int32 page_size = 5 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The value of + // [next_page_token][google.cloud.gkerecommender.v1.FetchProfilesResponse.next_page_token] + // received from a previous `FetchProfilesRequest` call. + // Provide this to retrieve the subsequent page in a multi-page list of + // results. When paginating, all other parameters provided to + // `FetchProfilesRequest` must match the call that provided the page + // token. + optional string page_token = 6 [(google.api.field_behavior) = OPTIONAL]; +} + +// Performance requirements for a profile and or model deployment. +message PerformanceRequirements { + // Optional. The target Normalized Time Per Output Token (NTPOT) in + // milliseconds. NTPOT is calculated as `request_latency / + // total_output_tokens`. If not provided, this target will not be enforced. + optional int32 target_ntpot_milliseconds = 1 + [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The target Time To First Token (TTFT) in milliseconds. TTFT is + // the time it takes to generate the first token for a request. If not + // provided, this target will not be enforced. + optional int32 target_ttft_milliseconds = 2 + [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The target cost for running a profile's model server. If not + // provided, this requirement will not be enforced. + Cost target_cost = 3 [(google.api.field_behavior) = OPTIONAL]; +} + +// Represents an amount of money in a specific currency. +message Amount { + // Output only. The whole units of the amount. + // For example if `currencyCode` is `"USD"`, then 1 unit is one US dollar. + int64 units = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Number of nano (10^-9) units of the amount. + // The value must be between -999,999,999 and +999,999,999 inclusive. + // If `units` is positive, `nanos` must be positive or zero. + // If `units` is zero, `nanos` can be positive, zero, or negative. + // If `units` is negative, `nanos` must be negative or zero. + // For example $-1.75 is represented as `units`=-1 and `nanos`=-750,000,000. + int32 nanos = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// Cost for running a model deployment on a given instance type. Currently, only +// USD currency code is supported. +message Cost { + // Optional. The cost per million output tokens, calculated as: + // $/output token = GPU $/s / (1/output-to-input-cost-ratio * input tokens/s + + // output tokens/s) + Amount cost_per_million_output_tokens = 1 + [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The cost per million input tokens. $/input token = ($/output + // token) / output-to-input-cost-ratio. + Amount cost_per_million_input_tokens = 2 + [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The pricing model used to calculate the cost. Can be one of: + // `3-years-cud`, `1-year-cud`, `on-demand`, `spot`. If not provided, `spot` + // will be used. + string pricing_model = 3 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The output-to-input cost ratio. This determines how the total GPU + // cost is split between input and output tokens. If not provided, `4.0` is + // used, assuming a 4:1 output:input cost ratio. + optional float output_input_cost_ratio = 4 + [(google.api.field_behavior) = OPTIONAL]; +} + +// Represents a range of throughput values in tokens per second. +message TokensPerSecondRange { + // Output only. The minimum value of the range. + int32 min = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The maximum value of the range. + int32 max = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// Represents a range of latency values in milliseconds. +message MillisecondRange { + // Output only. The minimum value of the range. + int32 min = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The maximum value of the range. + int32 max = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// Performance range for a model deployment. +message PerformanceRange { + // Output only. The range of throughput in output tokens per second. This is + // measured as total_output_tokens_generated_by_server / + // elapsed_time_in_seconds. + TokensPerSecondRange throughput_output_range = 1 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The range of TTFT (Time To First Token) in milliseconds. TTFT + // is the time it takes to generate the first token for a request. + MillisecondRange ttft_range = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The range of NTPOT (Normalized Time Per Output Token) in + // milliseconds. NTPOT is the request latency normalized by the number of + // output tokens, measured as request_latency / total_output_tokens. + MillisecondRange ntpot_range = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// Response message for +// [GkeInferenceQuickstart.FetchProfiles][google.cloud.gkerecommender.v1.GkeInferenceQuickstart.FetchProfiles]. +message FetchProfilesResponse { + // Output only. List of profiles that match the given model server info and + // performance requirements (if provided). + repeated Profile profile = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The combined range of performance values observed across all + // profiles in this response. + PerformanceRange performance_range = 2 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Additional comments related to the response. + string comments = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. A token which may be sent as + // [page_token][FetchProfilesResponse.page_token] in a subsequent + // `FetchProfilesResponse` call to retrieve the next page of results. If this + // field is omitted or empty, then there are no more results to return. + string next_page_token = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// Model server information gives. Valid model server info combinations can +// be found using +// [GkeInferenceQuickstart.FetchProfiles][google.cloud.gkerecommender.v1.GkeInferenceQuickstart.FetchProfiles]. +message ModelServerInfo { + // Required. The model. Open-source models follow the Huggingface Hub + // `owner/model_name` format. Use + // [GkeInferenceQuickstart.FetchModels][google.cloud.gkerecommender.v1.GkeInferenceQuickstart.FetchModels] + // to find available models. + string model = 1 [(google.api.field_behavior) = REQUIRED]; + + // Required. The model server. Open-source model servers use simplified, + // lowercase names (e.g., `vllm`). Use + // [GkeInferenceQuickstart.FetchModelServers][google.cloud.gkerecommender.v1.GkeInferenceQuickstart.FetchModelServers] + // to find available servers. + string model_server = 2 [(google.api.field_behavior) = REQUIRED]; + + // Optional. The model server version. Use + // [GkeInferenceQuickstart.FetchModelServerVersions][google.cloud.gkerecommender.v1.GkeInferenceQuickstart.FetchModelServerVersions] + // to find available versions. If not provided, the latest available version + // is used. + string model_server_version = 3 [(google.api.field_behavior) = OPTIONAL]; +} + +// Resources used by a model deployment. +message ResourcesUsed { + // Output only. The number of accelerators (e.g., GPUs or TPUs) used by the + // model deployment on the Kubernetes node. + int32 accelerator_count = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// Performance statistics for a model deployment. +message PerformanceStats { + // Output only. The number of queries per second. + // Note: This metric can vary widely based on context length and may not be a + // reliable measure of LLM throughput. + float queries_per_second = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The number of output tokens per second. This is the throughput + // measured as total_output_tokens_generated_by_server / + // elapsed_time_in_seconds. + int32 output_tokens_per_second = 2 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The Normalized Time Per Output Token (NTPOT) in milliseconds. + // This is the request latency normalized by the number of output tokens, + // measured as request_latency / total_output_tokens. + int32 ntpot_milliseconds = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The Time To First Token (TTFT) in milliseconds. This is the + // time it takes to generate the first token for a request. + int32 ttft_milliseconds = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The cost of running the model deployment. + repeated Cost cost = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// A profile containing information about a model deployment. +message Profile { + // Output only. The model server configuration. Use + // [GkeInferenceQuickstart.FetchProfiles][google.cloud.gkerecommender.v1.GkeInferenceQuickstart.FetchProfiles] + // to find valid configurations. + ModelServerInfo model_server_info = 1 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The accelerator type. Expected format: `nvidia-h100-80gb`. + string accelerator_type = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The TPU topology (if applicable). + string tpu_topology = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The instance type. Expected format: `a2-highgpu-1g`. + string instance_type = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The resources used by the model deployment. + ResourcesUsed resources_used = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The performance statistics for this profile. + repeated PerformanceStats performance_stats = 6 + [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// Request message for +// [GkeInferenceQuickstart.GenerateOptimizedManifest][google.cloud.gkerecommender.v1.GkeInferenceQuickstart.GenerateOptimizedManifest]. +message GenerateOptimizedManifestRequest { + // Required. The model server configuration to generate the manifest for. Use + // [GkeInferenceQuickstart.FetchProfiles][google.cloud.gkerecommender.v1.GkeInferenceQuickstart.FetchProfiles] + // to find valid configurations. + ModelServerInfo model_server_info = 1 + [(google.api.field_behavior) = REQUIRED]; + + // Required. The accelerator type. Use + // [GkeInferenceQuickstart.FetchProfiles][google.cloud.gkerecommender.v1.GkeInferenceQuickstart.FetchProfiles] + // to find valid accelerators for a given `model_server_info`. + string accelerator_type = 2 [(google.api.field_behavior) = REQUIRED]; + + // Optional. The kubernetes namespace to deploy the manifests in. + string kubernetes_namespace = 3 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The performance requirements to use for generating Horizontal Pod + // Autoscaler (HPA) resources. If provided, the manifest includes HPA + // resources to adjust the model server replica count to maintain the + // specified targets (e.g., NTPOT, TTFT) at a P50 latency. Cost targets are + // not currently supported for HPA generation. If the specified targets are + // not achievable, the HPA manifest will not be generated. + PerformanceRequirements performance_requirements = 4 + [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The storage configuration for the model. If not provided, the + // model is loaded from Huggingface. + StorageConfig storage_config = 5 [(google.api.field_behavior) = OPTIONAL]; +} + +// A Kubernetes manifest. +message KubernetesManifest { + // Output only. Kubernetes resource kind. + string kind = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Kubernetes API version. + string api_version = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. YAML content. + string content = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// Response message for +// [GkeInferenceQuickstart.GenerateOptimizedManifest][google.cloud.gkerecommender.v1.GkeInferenceQuickstart.GenerateOptimizedManifest]. +message GenerateOptimizedManifestResponse { + // Output only. A list of generated Kubernetes manifests. + repeated KubernetesManifest kubernetes_manifests = 1 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Comments related to deploying the generated manifests. + repeated string comments = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Additional information about the versioned dependencies used + // to generate the manifests. See [Run best practice inference with GKE + // Inference Quickstart + // recipes](https://cloud.google.com/kubernetes-engine/docs/how-to/machine-learning/inference/inference-quickstart) + // for details. + string manifest_version = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// Storage configuration for a model deployment. +message StorageConfig { + // Optional. The Google Cloud Storage bucket URI to load the model from. This + // URI must point to the directory containing the model's config file + // (`config.json`) and model weights. A tuned GCSFuse setup can improve + // LLM Pod startup time by more than 7x. Expected format: + // `gs:///`. + string model_bucket_uri = 1 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The URI for the GCS bucket containing the XLA compilation cache. + // If using TPUs, the XLA cache will be written to the same path as + // `model_bucket_uri`. This can speed up vLLM model preparation for repeated + // deployments. + string xla_cache_bucket_uri = 2 [(google.api.field_behavior) = OPTIONAL]; +} diff --git a/google/cloud/gkerecommender/v1/gkerecommender_grpc_service_config.json b/google/cloud/gkerecommender/v1/gkerecommender_grpc_service_config.json new file mode 100644 index 0000000000..de6695f622 --- /dev/null +++ b/google/cloud/gkerecommender/v1/gkerecommender_grpc_service_config.json @@ -0,0 +1,15 @@ +{ + "methodConfig": [ + { + "name": [ + { "service": "google.cloud.gkerecommender.v1.GkeInferenceQuickstart", "method": "FetchModels" }, + { "service": "google.cloud.gkerecommender.v1.GkeInferenceQuickstart", "method": "FetchModelServers" }, + { "service": "google.cloud.gkerecommender.v1.GkeInferenceQuickstart", "method": "FetchModelServerVersions" }, + { "service": "google.cloud.gkerecommender.v1.GkeInferenceQuickstart", "method": "FetchProfiles" }, + { "service": "google.cloud.gkerecommender.v1.GkeInferenceQuickstart", "method": "GenerateOptimizedManifest" }, + { "service": "google.cloud.gkerecommender.v1.GkeInferenceQuickstart", "method": "FetchBenchmarkingData" } + ], + "timeout": "60s" + } + ] +} \ No newline at end of file diff --git a/google/cloud/gkerecommender/v1/gkerecommender_v1.yaml b/google/cloud/gkerecommender/v1/gkerecommender_v1.yaml new file mode 100644 index 0000000000..7ce6b47ba9 --- /dev/null +++ b/google/cloud/gkerecommender/v1/gkerecommender_v1.yaml @@ -0,0 +1,17 @@ +type: google.api.Service +config_version: 3 +name: gkerecommender.googleapis.com +title: GKE Recommender API + +apis: +- name: google.cloud.gkerecommender.v1.GkeInferenceQuickstart + +documentation: + summary: GKE Recommender API + +authentication: + rules: + - selector: 'google.cloud.gkerecommender.v1.GkeInferenceQuickstart.*' + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform