mirror of
https://github.com/googleapis/googleapis.git
synced 2026-08-14 12:42:59 +02:00
609 lines
25 KiB
Protocol Buffer
609 lines
25 KiB
Protocol Buffer
// Copyright 2025 Google LLC
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
syntax = "proto3";
|
|
|
|
package google.cloud.dataproc.v1;
|
|
|
|
import "google/api/field_behavior.proto";
|
|
import "google/api/resource.proto";
|
|
import "google/protobuf/duration.proto";
|
|
import "google/protobuf/timestamp.proto";
|
|
|
|
option go_package = "cloud.google.com/go/dataproc/v2/apiv1/dataprocpb;dataprocpb";
|
|
option java_multiple_files = true;
|
|
option java_outer_classname = "SharedProto";
|
|
option java_package = "com.google.cloud.dataproc.v1";
|
|
option (google.api.resource_definition) = {
|
|
type: "container.googleapis.com/Cluster"
|
|
pattern: "projects/{project}/locations/{location}/clusters/{cluster}"
|
|
};
|
|
option (google.api.resource_definition) = {
|
|
type: "metastore.googleapis.com/Service"
|
|
pattern: "projects/{project}/locations/{location}/services/{service}"
|
|
};
|
|
option (google.api.resource_definition) = {
|
|
type: "cloudkms.googleapis.com/CryptoKey"
|
|
pattern: "projects/{project}/locations/{location}/keyRings/{key_ring}/cryptoKeys/{crypto_key}"
|
|
};
|
|
|
|
// Runtime configuration for a workload.
|
|
message RuntimeConfig {
|
|
// Optional. Version of the batch runtime.
|
|
string version = 1 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Optional custom container image for the job runtime environment.
|
|
// If not specified, a default container image will be used.
|
|
string container_image = 2 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. A mapping of property names to values, which are used to
|
|
// configure workload execution.
|
|
map<string, string> properties = 3 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Dependency repository configuration.
|
|
RepositoryConfig repository_config = 5
|
|
[(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Autotuning configuration of the workload.
|
|
AutotuningConfig autotuning_config = 6
|
|
[(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Cohort identifier. Identifies families of the workloads having
|
|
// the same shape, e.g. daily ETL jobs.
|
|
string cohort = 7 [(google.api.field_behavior) = OPTIONAL];
|
|
}
|
|
|
|
// Environment configuration for a workload.
|
|
message EnvironmentConfig {
|
|
// Optional. Execution configuration for a workload.
|
|
ExecutionConfig execution_config = 1 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Peripherals configuration that workload has access to.
|
|
PeripheralsConfig peripherals_config = 2
|
|
[(google.api.field_behavior) = OPTIONAL];
|
|
}
|
|
|
|
// Execution configuration for a workload.
|
|
message ExecutionConfig {
|
|
// Optional. Service account that used to execute workload.
|
|
string service_account = 2 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Network configuration for workload execution.
|
|
oneof network {
|
|
// Optional. Network URI to connect workload to.
|
|
string network_uri = 4 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Subnetwork URI to connect workload to.
|
|
string subnetwork_uri = 5 [(google.api.field_behavior) = OPTIONAL];
|
|
}
|
|
|
|
// Optional. Tags used for network traffic control.
|
|
repeated string network_tags = 6 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. The Cloud KMS key to use for encryption.
|
|
string kms_key = 7 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Applies to sessions only. The duration to keep the session alive
|
|
// while it's idling. Exceeding this threshold causes the session to
|
|
// terminate. This field cannot be set on a batch workload. Minimum value is
|
|
// 10 minutes; maximum value is 14 days (see JSON representation of
|
|
// [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json)).
|
|
// Defaults to 1 hour if not set.
|
|
// If both `ttl` and `idle_ttl` are specified for an interactive session,
|
|
// the conditions are treated as `OR` conditions: the workload will be
|
|
// terminated when it has been idle for `idle_ttl` or when `ttl` has been
|
|
// exceeded, whichever occurs first.
|
|
google.protobuf.Duration idle_ttl = 8
|
|
[(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. The duration after which the workload will be terminated,
|
|
// specified as the JSON representation for
|
|
// [Duration](https://protobuf.dev/programming-guides/proto3/#json).
|
|
// When the workload exceeds this duration, it will be unconditionally
|
|
// terminated without waiting for ongoing work to finish. If `ttl` is not
|
|
// specified for a batch workload, the workload will be allowed to run until
|
|
// it exits naturally (or run forever without exiting). If `ttl` is not
|
|
// specified for an interactive session, it defaults to 24 hours. If `ttl` is
|
|
// not specified for a batch that uses 2.1+ runtime version, it defaults to 4
|
|
// hours. Minimum value is 10 minutes; maximum value is 14 days. If both `ttl`
|
|
// and `idle_ttl` are specified (for an interactive session), the conditions
|
|
// are treated as `OR` conditions: the workload will be terminated when it has
|
|
// been idle for `idle_ttl` or when `ttl` has been exceeded, whichever occurs
|
|
// first.
|
|
google.protobuf.Duration ttl = 9 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. A Cloud Storage bucket used to stage workload dependencies,
|
|
// config files, and store workload output and other ephemeral data, such as
|
|
// Spark history files. If you do not specify a staging bucket, Cloud Dataproc
|
|
// will determine a Cloud Storage location according to the region where your
|
|
// workload is running, and then create and manage project-level, per-location
|
|
// staging and temporary buckets.
|
|
// **This field requires a Cloud Storage bucket name, not a `gs://...` URI to
|
|
// a Cloud Storage bucket.**
|
|
string staging_bucket = 10 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Authentication configuration used to set the default identity for
|
|
// the workload execution. The config specifies the type of identity
|
|
// (service account or user) that will be used by workloads to access
|
|
// resources on the project(s).
|
|
AuthenticationConfig authentication_config = 11
|
|
[(google.api.field_behavior) = OPTIONAL];
|
|
}
|
|
|
|
// Spark History Server configuration for the workload.
|
|
message SparkHistoryServerConfig {
|
|
// Optional. Resource name of an existing Dataproc Cluster to act as a Spark
|
|
// History Server for the workload.
|
|
//
|
|
// Example:
|
|
//
|
|
// * `projects/[project_id]/regions/[region]/clusters/[cluster_name]`
|
|
string dataproc_cluster = 1 [(google.api.field_behavior) = OPTIONAL];
|
|
}
|
|
|
|
// Auxiliary services configuration for a workload.
|
|
message PeripheralsConfig {
|
|
// Optional. Resource name of an existing Dataproc Metastore service.
|
|
//
|
|
// Example:
|
|
//
|
|
// * `projects/[project_id]/locations/[region]/services/[service_id]`
|
|
string metastore_service = 1 [
|
|
(google.api.field_behavior) = OPTIONAL,
|
|
(google.api.resource_reference) = {
|
|
type: "metastore.googleapis.com/Service"
|
|
}
|
|
];
|
|
|
|
// Optional. The Spark History Server configuration for the workload.
|
|
SparkHistoryServerConfig spark_history_server_config = 2
|
|
[(google.api.field_behavior) = OPTIONAL];
|
|
}
|
|
|
|
// Runtime information about workload execution.
|
|
message RuntimeInfo {
|
|
// Output only. Map of remote access endpoints (such as web interfaces and
|
|
// APIs) to their URIs.
|
|
map<string, string> endpoints = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
|
|
|
|
// Output only. A URI pointing to the location of the stdout and stderr of the
|
|
// workload.
|
|
string output_uri = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
|
|
|
|
// Output only. A URI pointing to the location of the diagnostics tarball.
|
|
string diagnostic_output_uri = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
|
|
|
|
// Output only. Approximate workload resource usage, calculated when
|
|
// the workload completes (see [Dataproc Serverless pricing]
|
|
// (https://cloud.google.com/dataproc-serverless/pricing)).
|
|
//
|
|
// **Note:** This metric calculation may change in the future, for
|
|
// example, to capture cumulative workload resource
|
|
// consumption during workload execution (see the
|
|
// [Dataproc Serverless release notes]
|
|
// (https://cloud.google.com/dataproc-serverless/docs/release-notes)
|
|
// for announcements, changes, fixes
|
|
// and other Dataproc developments).
|
|
UsageMetrics approximate_usage = 6
|
|
[(google.api.field_behavior) = OUTPUT_ONLY];
|
|
|
|
// Output only. Snapshot of current workload resource usage.
|
|
UsageSnapshot current_usage = 7 [(google.api.field_behavior) = OUTPUT_ONLY];
|
|
}
|
|
|
|
// Usage metrics represent approximate total resources consumed by a workload.
|
|
message UsageMetrics {
|
|
// Optional. DCU (Dataproc Compute Units) usage in (`milliDCU` x `seconds`)
|
|
// (see [Dataproc Serverless pricing]
|
|
// (https://cloud.google.com/dataproc-serverless/pricing)).
|
|
int64 milli_dcu_seconds = 1 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Shuffle storage usage in (`GB` x `seconds`) (see
|
|
// [Dataproc Serverless pricing]
|
|
// (https://cloud.google.com/dataproc-serverless/pricing)).
|
|
int64 shuffle_storage_gb_seconds = 2 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Accelerator usage in (`milliAccelerator` x `seconds`) (see
|
|
// [Dataproc Serverless pricing]
|
|
// (https://cloud.google.com/dataproc-serverless/pricing)).
|
|
int64 milli_accelerator_seconds = 3 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Accelerator type being used, if any
|
|
string accelerator_type = 4 [(google.api.field_behavior) = OPTIONAL];
|
|
}
|
|
|
|
// The usage snapshot represents the resources consumed by a workload at a
|
|
// specified time.
|
|
message UsageSnapshot {
|
|
// Optional. Milli (one-thousandth) Dataproc Compute Units (DCUs) (see
|
|
// [Dataproc Serverless pricing]
|
|
// (https://cloud.google.com/dataproc-serverless/pricing)).
|
|
int64 milli_dcu = 1 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Shuffle Storage in gigabytes (GB). (see [Dataproc Serverless
|
|
// pricing] (https://cloud.google.com/dataproc-serverless/pricing))
|
|
int64 shuffle_storage_gb = 2 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Milli (one-thousandth) Dataproc Compute Units (DCUs) charged at
|
|
// premium tier (see [Dataproc Serverless pricing]
|
|
// (https://cloud.google.com/dataproc-serverless/pricing)).
|
|
int64 milli_dcu_premium = 4 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Shuffle Storage in gigabytes (GB) charged at premium tier. (see
|
|
// [Dataproc Serverless pricing]
|
|
// (https://cloud.google.com/dataproc-serverless/pricing))
|
|
int64 shuffle_storage_gb_premium = 5 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Milli (one-thousandth) accelerator. (see [Dataproc
|
|
// Serverless pricing] (https://cloud.google.com/dataproc-serverless/pricing))
|
|
int64 milli_accelerator = 6 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Accelerator type being used, if any
|
|
string accelerator_type = 7 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. The timestamp of the usage snapshot.
|
|
google.protobuf.Timestamp snapshot_time = 3
|
|
[(google.api.field_behavior) = OPTIONAL];
|
|
}
|
|
|
|
// The cluster's GKE config.
|
|
message GkeClusterConfig {
|
|
// Optional. A target GKE cluster to deploy to. It must be in the same project
|
|
// and region as the Dataproc cluster (the GKE cluster can be zonal or
|
|
// regional). Format:
|
|
// 'projects/{project}/locations/{location}/clusters/{cluster_id}'
|
|
string gke_cluster_target = 2 [
|
|
(google.api.field_behavior) = OPTIONAL,
|
|
(google.api.resource_reference) = {
|
|
type: "container.googleapis.com/Cluster"
|
|
}
|
|
];
|
|
|
|
// Optional. GKE node pools where workloads will be scheduled. At least one
|
|
// node pool must be assigned the `DEFAULT`
|
|
// [GkeNodePoolTarget.Role][google.cloud.dataproc.v1.GkeNodePoolTarget.Role].
|
|
// If a `GkeNodePoolTarget` is not specified, Dataproc constructs a `DEFAULT`
|
|
// `GkeNodePoolTarget`. Each role can be given to only one
|
|
// `GkeNodePoolTarget`. All node pools must have the same location settings.
|
|
repeated GkeNodePoolTarget node_pool_target = 3
|
|
[(google.api.field_behavior) = OPTIONAL];
|
|
}
|
|
|
|
// The configuration for running the Dataproc cluster on Kubernetes.
|
|
message KubernetesClusterConfig {
|
|
// Optional. A namespace within the Kubernetes cluster to deploy into. If this
|
|
// namespace does not exist, it is created. If it exists, Dataproc verifies
|
|
// that another Dataproc VirtualCluster is not installed into it. If not
|
|
// specified, the name of the Dataproc Cluster is used.
|
|
string kubernetes_namespace = 1 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
oneof config {
|
|
// Required. The configuration for running the Dataproc cluster on GKE.
|
|
GkeClusterConfig gke_cluster_config = 2
|
|
[(google.api.field_behavior) = REQUIRED];
|
|
}
|
|
|
|
// Optional. The software configuration for this Dataproc cluster running on
|
|
// Kubernetes.
|
|
KubernetesSoftwareConfig kubernetes_software_config = 3
|
|
[(google.api.field_behavior) = OPTIONAL];
|
|
}
|
|
|
|
// The software configuration for this Dataproc cluster running on Kubernetes.
|
|
message KubernetesSoftwareConfig {
|
|
// The components that should be installed in this Dataproc cluster. The key
|
|
// must be a string from the KubernetesComponent enumeration. The value is
|
|
// the version of the software to be installed.
|
|
// At least one entry must be specified.
|
|
map<string, string> component_version = 1;
|
|
|
|
// The properties to set on daemon config files.
|
|
//
|
|
// Property keys are specified in `prefix:property` format, for example
|
|
// `spark:spark.kubernetes.container.image`. The following are supported
|
|
// prefixes and their mappings:
|
|
//
|
|
// * spark: `spark-defaults.conf`
|
|
//
|
|
// For more information, see [Cluster
|
|
// properties](https://cloud.google.com/dataproc/docs/concepts/cluster-properties).
|
|
map<string, string> properties = 2;
|
|
}
|
|
|
|
// GKE node pools that Dataproc workloads run on.
|
|
message GkeNodePoolTarget {
|
|
// `Role` specifies the tasks that will run on the node pool. Roles can be
|
|
// specific to workloads. Exactly one
|
|
// [GkeNodePoolTarget][google.cloud.dataproc.v1.GkeNodePoolTarget] within the
|
|
// virtual cluster must have the `DEFAULT` role, which is used to run all
|
|
// workloads that are not associated with a node pool.
|
|
enum Role {
|
|
// Role is unspecified.
|
|
ROLE_UNSPECIFIED = 0;
|
|
|
|
// At least one node pool must have the `DEFAULT` role.
|
|
// Work assigned to a role that is not associated with a node pool
|
|
// is assigned to the node pool with the `DEFAULT` role. For example,
|
|
// work assigned to the `CONTROLLER` role will be assigned to the node pool
|
|
// with the `DEFAULT` role if no node pool has the `CONTROLLER` role.
|
|
DEFAULT = 1;
|
|
|
|
// Run work associated with the Dataproc control plane (for example,
|
|
// controllers and webhooks). Very low resource requirements.
|
|
CONTROLLER = 2;
|
|
|
|
// Run work associated with a Spark driver of a job.
|
|
SPARK_DRIVER = 3;
|
|
|
|
// Run work associated with a Spark executor of a job.
|
|
SPARK_EXECUTOR = 4;
|
|
}
|
|
|
|
// Required. The target GKE node pool.
|
|
// Format:
|
|
// 'projects/{project}/locations/{location}/clusters/{cluster}/nodePools/{node_pool}'
|
|
string node_pool = 1 [(google.api.field_behavior) = REQUIRED];
|
|
|
|
// Required. The roles associated with the GKE node pool.
|
|
repeated Role roles = 2 [(google.api.field_behavior) = REQUIRED];
|
|
|
|
// Input only. The configuration for the GKE node pool.
|
|
//
|
|
// If specified, Dataproc attempts to create a node pool with the
|
|
// specified shape. If one with the same name already exists, it is
|
|
// verified against all specified fields. If a field differs, the
|
|
// virtual cluster creation will fail.
|
|
//
|
|
// If omitted, any node pool with the specified name is used. If a
|
|
// node pool with the specified name does not exist, Dataproc create a
|
|
// node pool with default values.
|
|
//
|
|
// This is an input only field. It will not be returned by the API.
|
|
GkeNodePoolConfig node_pool_config = 3
|
|
[(google.api.field_behavior) = INPUT_ONLY];
|
|
}
|
|
|
|
// The configuration of a GKE node pool used by a [Dataproc-on-GKE
|
|
// cluster](https://cloud.google.com/dataproc/docs/concepts/jobs/dataproc-gke#create-a-dataproc-on-gke-cluster).
|
|
message GkeNodePoolConfig {
|
|
// Parameters that describe cluster nodes.
|
|
message GkeNodeConfig {
|
|
// Optional. The name of a Compute Engine [machine
|
|
// type](https://cloud.google.com/compute/docs/machine-types).
|
|
string machine_type = 1 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. The number of local SSD disks to attach to the node, which is
|
|
// limited by the maximum number of disks allowable per zone (see [Adding
|
|
// Local SSDs](https://cloud.google.com/compute/docs/disks/local-ssd)).
|
|
int32 local_ssd_count = 7 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Whether the nodes are created as legacy [preemptible VM
|
|
// instances] (https://cloud.google.com/compute/docs/instances/preemptible).
|
|
// Also see
|
|
// [Spot][google.cloud.dataproc.v1.GkeNodePoolConfig.GkeNodeConfig.spot]
|
|
// VMs, preemptible VM instances without a maximum lifetime. Legacy and Spot
|
|
// preemptible nodes cannot be used in a node pool with the `CONTROLLER`
|
|
// [role]
|
|
// (/dataproc/docs/reference/rest/v1/projects.regions.clusters#role)
|
|
// or in the DEFAULT node pool if the CONTROLLER role is not assigned (the
|
|
// DEFAULT node pool will assume the CONTROLLER role).
|
|
bool preemptible = 10 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. A list of [hardware
|
|
// accelerators](https://cloud.google.com/compute/docs/gpus) to attach to
|
|
// each node.
|
|
repeated GkeNodePoolAcceleratorConfig accelerators = 11
|
|
[(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. [Minimum CPU
|
|
// platform](https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform)
|
|
// to be used by this instance. The instance may be scheduled on the
|
|
// specified or a newer CPU platform. Specify the friendly names of CPU
|
|
// platforms, such as "Intel Haswell"` or Intel Sandy Bridge".
|
|
string min_cpu_platform = 13 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. The [Customer Managed Encryption Key (CMEK)]
|
|
// (https://cloud.google.com/kubernetes-engine/docs/how-to/using-cmek)
|
|
// used to encrypt the boot disk attached to each node in the node pool.
|
|
// Specify the key using the following format:
|
|
// <code>projects/<var>KEY_PROJECT_ID</var>/locations/<var>LOCATION</var>/keyRings/<var>RING_NAME</var>/cryptoKeys/<var>KEY_NAME</var></code>.
|
|
string boot_disk_kms_key = 23 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Whether the nodes are created as [Spot VM instances]
|
|
// (https://cloud.google.com/compute/docs/instances/spot).
|
|
// Spot VMs are the latest update to legacy
|
|
// [preemptible
|
|
// VMs][google.cloud.dataproc.v1.GkeNodePoolConfig.GkeNodeConfig.preemptible].
|
|
// Spot VMs do not have a maximum lifetime. Legacy and Spot preemptible
|
|
// nodes cannot be used in a node pool with the `CONTROLLER`
|
|
// [role](/dataproc/docs/reference/rest/v1/projects.regions.clusters#role)
|
|
// or in the DEFAULT node pool if the CONTROLLER role is not assigned (the
|
|
// DEFAULT node pool will assume the CONTROLLER role).
|
|
bool spot = 32 [(google.api.field_behavior) = OPTIONAL];
|
|
}
|
|
|
|
// A GkeNodeConfigAcceleratorConfig represents a Hardware Accelerator request
|
|
// for a node pool.
|
|
message GkeNodePoolAcceleratorConfig {
|
|
// The number of accelerator cards exposed to an instance.
|
|
int64 accelerator_count = 1;
|
|
|
|
// The accelerator type resource namename (see GPUs on Compute Engine).
|
|
string accelerator_type = 2;
|
|
|
|
// Size of partitions to create on the GPU. Valid values are described in
|
|
// the NVIDIA [mig user
|
|
// guide](https://docs.nvidia.com/datacenter/tesla/mig-user-guide/#partitioning).
|
|
string gpu_partition_size = 3;
|
|
}
|
|
|
|
// GkeNodePoolAutoscaling contains information the cluster autoscaler needs to
|
|
// adjust the size of the node pool to the current cluster usage.
|
|
message GkeNodePoolAutoscalingConfig {
|
|
// The minimum number of nodes in the node pool. Must be >= 0 and <=
|
|
// max_node_count.
|
|
int32 min_node_count = 2;
|
|
|
|
// The maximum number of nodes in the node pool. Must be >= min_node_count,
|
|
// and must be > 0.
|
|
// **Note:** Quota must be sufficient to scale up the cluster.
|
|
int32 max_node_count = 3;
|
|
}
|
|
|
|
// Optional. The node pool configuration.
|
|
GkeNodeConfig config = 2 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. The list of Compute Engine
|
|
// [zones](https://cloud.google.com/compute/docs/zones#available) where
|
|
// node pool nodes associated with a Dataproc on GKE virtual cluster
|
|
// will be located.
|
|
//
|
|
// **Note:** All node pools associated with a virtual cluster
|
|
// must be located in the same region as the virtual cluster, and they must
|
|
// be located in the same zone within that region.
|
|
//
|
|
// If a location is not specified during node pool creation, Dataproc on GKE
|
|
// will choose the zone.
|
|
repeated string locations = 13 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. The autoscaler configuration for this node pool. The autoscaler
|
|
// is enabled only when a valid configuration is present.
|
|
GkeNodePoolAutoscalingConfig autoscaling = 4
|
|
[(google.api.field_behavior) = OPTIONAL];
|
|
}
|
|
|
|
// Authentication configuration for a workload is used to set the default
|
|
// identity for the workload execution.
|
|
// The config specifies the type of identity (service account or user) that
|
|
// will be used by workloads to access resources on the project(s).
|
|
message AuthenticationConfig {
|
|
// Authentication types for workload execution.
|
|
enum AuthenticationType {
|
|
// If AuthenticationType is unspecified then END_USER_CREDENTIALS is used
|
|
// for 3.0 and newer runtimes, and SERVICE_ACCOUNT is used for older
|
|
// runtimes.
|
|
AUTHENTICATION_TYPE_UNSPECIFIED = 0;
|
|
|
|
// Use service account credentials for authenticating to other services.
|
|
SERVICE_ACCOUNT = 1;
|
|
|
|
// Use OAuth credentials associated with the workload creator/user for
|
|
// authenticating to other services.
|
|
END_USER_CREDENTIALS = 2;
|
|
}
|
|
|
|
// Optional. Authentication type for the user workload running in containers.
|
|
AuthenticationType user_workload_authentication_type = 1
|
|
[(google.api.field_behavior) = OPTIONAL];
|
|
}
|
|
|
|
// Autotuning configuration of the workload.
|
|
message AutotuningConfig {
|
|
// Scenario represents a specific goal that autotuning will attempt to achieve
|
|
// by modifying workloads.
|
|
enum Scenario {
|
|
// Default value.
|
|
SCENARIO_UNSPECIFIED = 0;
|
|
|
|
// Scaling recommendations such as initialExecutors.
|
|
SCALING = 2;
|
|
|
|
// Adding hints for potential relation broadcasts.
|
|
BROADCAST_HASH_JOIN = 3;
|
|
|
|
// Memory management for workloads.
|
|
MEMORY = 4;
|
|
}
|
|
|
|
// Optional. Scenarios for which tunings are applied.
|
|
repeated Scenario scenarios = 2 [(google.api.field_behavior) = OPTIONAL];
|
|
}
|
|
|
|
// Configuration for dependency repositories
|
|
message RepositoryConfig {
|
|
// Optional. Configuration for PyPi repository.
|
|
PyPiRepositoryConfig pypi_repository_config = 1
|
|
[(google.api.field_behavior) = OPTIONAL];
|
|
}
|
|
|
|
// Configuration for PyPi repository
|
|
message PyPiRepositoryConfig {
|
|
// Optional. PyPi repository address
|
|
string pypi_repository = 1 [(google.api.field_behavior) = OPTIONAL];
|
|
}
|
|
|
|
// Cluster components that can be activated.
|
|
enum Component {
|
|
// Unspecified component. Specifying this will cause Cluster creation to fail.
|
|
COMPONENT_UNSPECIFIED = 0;
|
|
|
|
// The Anaconda component is no longer supported or applicable to
|
|
// [supported Dataproc on Compute Engine image versions]
|
|
// (https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-version-clusters#supported-dataproc-image-versions).
|
|
// It cannot be activated on clusters created with supported Dataproc on
|
|
// Compute Engine image versions.
|
|
ANACONDA = 5;
|
|
|
|
// Docker
|
|
DOCKER = 13;
|
|
|
|
// The Druid query engine. (alpha)
|
|
DRUID = 9;
|
|
|
|
// Flink
|
|
FLINK = 14;
|
|
|
|
// HBase. (beta)
|
|
HBASE = 11;
|
|
|
|
// The Hive Web HCatalog (the REST service for accessing HCatalog).
|
|
HIVE_WEBHCAT = 3;
|
|
|
|
// Hudi.
|
|
HUDI = 18;
|
|
|
|
// The Jupyter Notebook.
|
|
JUPYTER = 1;
|
|
|
|
// The Presto query engine.
|
|
PRESTO = 6;
|
|
|
|
// The Trino query engine.
|
|
TRINO = 17;
|
|
|
|
// The Ranger service.
|
|
RANGER = 12;
|
|
|
|
// The Solr service.
|
|
SOLR = 10;
|
|
|
|
// The Zeppelin notebook.
|
|
ZEPPELIN = 4;
|
|
|
|
// The Zookeeper service.
|
|
ZOOKEEPER = 8;
|
|
}
|
|
|
|
// Actions in response to failure of a resource associated with a cluster.
|
|
enum FailureAction {
|
|
// When FailureAction is unspecified, failure action defaults to NO_ACTION.
|
|
FAILURE_ACTION_UNSPECIFIED = 0;
|
|
|
|
// Take no action on failure to create a cluster resource. NO_ACTION is the
|
|
// default.
|
|
NO_ACTION = 1;
|
|
|
|
// Delete the failed cluster resource.
|
|
DELETE = 2;
|
|
}
|