From 439f098cb730af18c8ae4e65971ea3badf45138a Mon Sep 17 00:00:00 2001 From: Google APIs Date: Thu, 8 Apr 2021 15:22:23 -0700 Subject: [PATCH] feat: update the Dataproc V1 API client library This includes the following: 1. The new start and stop cluster methods. 2. The ability to specify a metastore config in a cluster. 3. The ability to specify a (BETA) GKE cluster when creating a Dataproc cluster. 4. The ability to configure the behavior for private IPv6 cluster networking. 5. The ability to specify node affinity groups for clusters. 6. The ability to specify shielded VM configurations for clusters. 7. Support for service-account based secure multi-tenancy. 8. The ability to specify cluster labels for picking which cluster should run a job. 9. Components for DOCKER, DRUID, FLINK, HBASE, RANGER, and SOLR 10. The ability to specify a DAG timeout for workflows. Committer: @ojarjur PiperOrigin-RevId: 367518225 --- google/cloud/dataproc/v1/clusters.proto | 272 ++++++++++++++++-- google/cloud/dataproc/v1/dataproc_v1.yaml | 2 +- google/cloud/dataproc/v1/jobs.proto | 70 +++-- google/cloud/dataproc/v1/shared.proto | 26 +- .../dataproc/v1/workflow_templates.proto | 110 +++++-- 5 files changed, 408 insertions(+), 72 deletions(-) diff --git a/google/cloud/dataproc/v1/clusters.proto b/google/cloud/dataproc/v1/clusters.proto index ccff3522a8..147c95b7c5 100644 --- a/google/cloud/dataproc/v1/clusters.proto +++ b/google/cloud/dataproc/v1/clusters.proto @@ -19,6 +19,7 @@ package google.cloud.dataproc.v1; import "google/api/annotations.proto"; import "google/api/client.proto"; import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; import "google/cloud/dataproc/v1/shared.proto"; import "google/longrunning/operations.proto"; import "google/protobuf/duration.proto"; @@ -29,6 +30,14 @@ option go_package = "google.golang.org/genproto/googleapis/cloud/dataproc/v1;dat option java_multiple_files = true; option java_outer_classname = "ClustersProto"; option java_package = "com.google.cloud.dataproc.v1"; +option (google.api.resource_definition) = { + type: "container.googleapis.com/Cluster" + pattern: "projects/{project}/locations/{location}/clusters/{cluster}" +}; +option (google.api.resource_definition) = { + type: "metastore.googleapis.com/Service" + pattern: "projects/{project}/locations/{location}/services/{service}" +}; // The ClusterControllerService provides methods to manage clusters // of Compute Engine instances. @@ -66,6 +75,30 @@ service ClusterController { }; } + // Stops a cluster in a project. + rpc StopCluster(StopClusterRequest) returns (google.longrunning.Operation) { + option (google.api.http) = { + post: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}:stop" + body: "*" + }; + option (google.longrunning.operation_info) = { + response_type: "Cluster" + metadata_type: "google.cloud.dataproc.v1.ClusterOperationMetadata" + }; + } + + // Starts a cluster in a project. + rpc StartCluster(StartClusterRequest) returns (google.longrunning.Operation) { + option (google.api.http) = { + post: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}:start" + body: "*" + }; + option (google.longrunning.operation_info) = { + response_type: "Cluster" + metadata_type: "google.cloud.dataproc.v1.ClusterOperationMetadata" + }; + } + // Deletes a cluster in a project. The returned // [Operation.metadata][google.longrunning.Operation.metadata] will be // [ClusterOperationMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata). @@ -168,6 +201,8 @@ message ClusterConfig { // and manage this project-level, per-location bucket (see // [Dataproc staging // bucket](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/staging-bucket)). + // **This field requires a Cloud Storage bucket name, not a URI to a Cloud + // Storage bucket.** string config_bucket = 1 [(google.api.field_behavior) = OPTIONAL]; // Optional. A Cloud Storage bucket used to store ephemeral cluster and jobs data, @@ -179,6 +214,8 @@ message ClusterConfig { // and manage this project-level, per-location bucket. The default bucket has // a TTL of 90 days, but you can use any TTL (or none) if you specify a // bucket. + // **This field requires a Cloud Storage bucket name, not a URI to a Cloud + // Storage bucket.** string temp_bucket = 2 [(google.api.field_behavior) = OPTIONAL]; // Optional. The shared Compute Engine config settings for @@ -230,6 +267,36 @@ message ClusterConfig { // Optional. Port/endpoint configuration for this cluster EndpointConfig endpoint_config = 19 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Metastore configuration. + MetastoreConfig metastore_config = 20 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. BETA. The Kubernetes Engine config for Dataproc clusters deployed to + // Kubernetes. Setting this is considered mutually exclusive with Compute + // Engine-based options such as `gce_cluster_config`, `master_config`, + // `worker_config`, `secondary_worker_config`, and `autoscaling_config`. + GkeClusterConfig gke_cluster_config = 21 [(google.api.field_behavior) = OPTIONAL]; +} + +// The GKE config for this cluster. +message GkeClusterConfig { + // A full, namespace-isolated deployment target for an existing GKE cluster. + message NamespacedGkeDeploymentTarget { + // Optional. The target GKE cluster to deploy to. + // Format: 'projects/{project}/locations/{location}/clusters/{cluster_id}' + string target_gke_cluster = 1 [ + (google.api.field_behavior) = OPTIONAL, + (google.api.resource_reference) = { + type: "container.googleapis.com/Cluster" + } + ]; + + // Optional. A namespace within the GKE cluster to deploy into. + string cluster_namespace = 2 [(google.api.field_behavior) = OPTIONAL]; + } + + // Optional. A target for the deployment. + NamespacedGkeDeploymentTarget namespaced_gke_deployment_target = 1 [(google.api.field_behavior) = OPTIONAL]; } // Endpoint config for this cluster @@ -267,6 +334,30 @@ message EncryptionConfig { // Common config settings for resources of Compute Engine cluster // instances, applicable to all instances in the cluster. message GceClusterConfig { + // `PrivateIpv6GoogleAccess` controls whether and how Dataproc cluster nodes + // can communicate with Google Services through gRPC over IPv6. + // These values are directly mapped to corresponding values in the + // [Compute Engine Instance + // fields](https://cloud.google.com/compute/docs/reference/rest/v1/instances). + enum PrivateIpv6GoogleAccess { + // If unspecified, Compute Engine default behavior will apply, which + // is the same as [INHERIT_FROM_SUBNETWORK][google.cloud.dataproc.v1.GceClusterConfig.PrivateIpv6GoogleAccess.INHERIT_FROM_SUBNETWORK]. + PRIVATE_IPV6_GOOGLE_ACCESS_UNSPECIFIED = 0; + + // Private access to and from Google Services configuration + // inherited from the subnetwork configuration. This is the + // default Compute Engine behavior. + INHERIT_FROM_SUBNETWORK = 1; + + // Enables outbound private IPv6 access to Google Services from the Dataproc + // cluster. + OUTBOUND = 2; + + // Enables bidirectional private IPv6 access between Google Services and the + // Dataproc cluster. + BIDIRECTIONAL = 3; + } + // Optional. The zone where the Compute Engine cluster will be located. // On a create request, it is required in the "global" region. If omitted // in a non-global Dataproc region, the service will pick a zone in the @@ -312,6 +403,9 @@ message GceClusterConfig { // configured to be accessible without external IP addresses. bool internal_ip_only = 7 [(google.api.field_behavior) = OPTIONAL]; + // Optional. The type of IPv6 access for a cluster. + PrivateIpv6GoogleAccess private_ipv6_google_access = 12 [(google.api.field_behavior) = OPTIONAL]; + // Optional. The [Dataproc service // account](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/service-accounts#service_accounts_in_dataproc) // (also see [VM Data Plane @@ -352,6 +446,41 @@ message GceClusterConfig { // Optional. Reservation Affinity for consuming Zonal reservation. ReservationAffinity reservation_affinity = 11 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Node Group Affinity for sole-tenant clusters. + NodeGroupAffinity node_group_affinity = 13 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Shielded Instance Config for clusters using [Compute Engine Shielded + // VMs](https://cloud.google.com/security/shielded-cloud/shielded-vm). + ShieldedInstanceConfig shielded_instance_config = 14 [(google.api.field_behavior) = OPTIONAL]; +} + +// Node Group Affinity for clusters using sole-tenant node groups. +message NodeGroupAffinity { + // Required. The URI of a + // sole-tenant [node group + // resource](https://cloud.google.com/compute/docs/reference/rest/v1/nodeGroups) + // that the cluster will be created on. + // + // A full URL, partial URI, or node group name are valid. Examples: + // + // * `https://www.googleapis.com/compute/v1/projects/[project_id]/zones/us-central1-a/nodeGroups/node-group-1` + // * `projects/[project_id]/zones/us-central1-a/nodeGroups/node-group-1` + // * `node-group-1` + string node_group_uri = 1 [(google.api.field_behavior) = REQUIRED]; +} + +// Shielded Instance Config for clusters using [Compute Engine Shielded +// VMs](https://cloud.google.com/security/shielded-cloud/shielded-vm). +message ShieldedInstanceConfig { + // Optional. Defines whether instances have Secure Boot enabled. + bool enable_secure_boot = 1 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Defines whether instances have the vTPM enabled. + bool enable_vtpm = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Defines whether instances have integrity monitoring enabled. + bool enable_integrity_monitoring = 3 [(google.api.field_behavior) = OPTIONAL]; } // The config settings for Compute Engine resources in @@ -379,7 +508,11 @@ message InstanceGroupConfig { } // Optional. The number of VM instances in the instance group. - // For master instance groups, must be set to 1. + // For [HA + // cluster](/dataproc/docs/concepts/configuring-clusters/high-availability) + // [master_config](#FIELDS.master_config) groups, **must be set to 3**. + // For standard cluster [master_config](#FIELDS.master_config) groups, + // **must be set to 1**. int32 num_instances = 1 [(google.api.field_behavior) = OPTIONAL]; // Output only. The list of instance names. Dataproc derives the names @@ -491,8 +624,10 @@ message AcceleratorConfig { // Specifies the config of disk options for a group of VM instances. message DiskConfig { // Optional. Type of the boot disk (default is "pd-standard"). - // Valid values: "pd-ssd" (Persistent Disk Solid State Drive) or - // "pd-standard" (Persistent Disk Hard Disk Drive). + // Valid values: "pd-balanced" (Persistent Disk Balanced Solid State Drive), + // "pd-ssd" (Persistent Disk Solid State Drive), + // or "pd-standard" (Persistent Disk Hard Disk Drive). + // See [Disk types](https://cloud.google.com/compute/docs/disks#disk-types). string boot_disk_type = 3 [(google.api.field_behavior) = OPTIONAL]; // Optional. Size in GB of the boot disk (default is 500GB). @@ -544,6 +679,15 @@ message ClusterStatus { // The cluster is being updated. It continues to accept and process jobs. UPDATING = 5; + + // The cluster is being stopped. It cannot be used. + STOPPING = 6; + + // The cluster is currently stopped. It is not ready for use. + STOPPED = 7; + + // The cluster is being started. It is not ready for use. + STARTING = 8; } // The cluster substate. @@ -583,10 +727,14 @@ message ClusterStatus { Substate substate = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; } -// Security related configuration, including Kerberos. +// Security related configuration, including encryption, Kerberos, etc. message SecurityConfig { - // Kerberos related configuration. - KerberosConfig kerberos_config = 1; + // Optional. Kerberos related configuration. + KerberosConfig kerberos_config = 1 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Identity related configuration, including service account based + // secure multi-tenancy user mappings. + IdentityConfig identity_config = 2 [(google.api.field_behavior) = OPTIONAL]; } // Specifies Kerberos related configuration. @@ -595,13 +743,13 @@ message KerberosConfig { // this field to true to enable Kerberos on a cluster. bool enable_kerberos = 1 [(google.api.field_behavior) = OPTIONAL]; - // Required. The Cloud Storage URI of a KMS encrypted file containing the root + // Optional. The Cloud Storage URI of a KMS encrypted file containing the root // principal password. - string root_principal_password_uri = 2 [(google.api.field_behavior) = REQUIRED]; + string root_principal_password_uri = 2 [(google.api.field_behavior) = OPTIONAL]; - // Required. The uri of the KMS key used to encrypt various sensitive + // Optional. The uri of the KMS key used to encrypt various sensitive // files. - string kms_key_uri = 3 [(google.api.field_behavior) = REQUIRED]; + string kms_key_uri = 3 [(google.api.field_behavior) = OPTIONAL]; // Optional. The Cloud Storage URI of the keystore file used for SSL // encryption. If not provided, Dataproc will provide a self-signed @@ -659,6 +807,13 @@ message KerberosConfig { string realm = 15 [(google.api.field_behavior) = OPTIONAL]; } +// Identity related configuration, including service account based +// secure multi-tenancy user mappings. +message IdentityConfig { + // Required. Map of user to service account. + map user_service_account_mapping = 1 [(google.api.field_behavior) = REQUIRED]; +} + // Specifies the selection and config of software inside the cluster. message SoftwareConfig { // Optional. The version of software inside the cluster. It must be one of the @@ -698,9 +853,9 @@ message SoftwareConfig { message LifecycleConfig { // Optional. The duration to keep the cluster alive while idling (when no jobs // are running). Passing this threshold will cause the cluster to be - // deleted. Minimum value is 10 minutes; maximum value is 14 days (see JSON + // deleted. Minimum value is 5 minutes; maximum value is 14 days (see JSON // representation of - // [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json). + // [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json)). google.protobuf.Duration idle_delete_ttl = 1 [(google.api.field_behavior) = OPTIONAL]; // Either the exact time the cluster should be deleted at or @@ -724,6 +879,21 @@ message LifecycleConfig { google.protobuf.Timestamp idle_start_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; } +// Specifies a Metastore configuration. +message MetastoreConfig { + // Required. Resource name of an existing Dataproc Metastore service. + // + // Example: + // + // * `projects/[project_id]/locations/[dataproc_region]/services/[service-name]` + string dataproc_metastore_service = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "metastore.googleapis.com/Service" + } + ]; +} + // Contains cluster daemon metrics, such as HDFS and YARN stats. // // **Beta Feature**: This report is available for testing purposes only. It may @@ -748,9 +918,9 @@ message CreateClusterRequest { // Required. The cluster to create. Cluster cluster = 2 [(google.api.field_behavior) = REQUIRED]; - // Optional. A unique id used to identify the request. If the server - // receives two [CreateClusterRequest][google.cloud.dataproc.v1.CreateClusterRequest] requests with the same - // id, then the second request will be ignored and the + // Optional. A unique id used to identify the request. If the server receives two + // [CreateClusterRequest](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#google.cloud.dataproc.v1.CreateClusterRequest)s + // with the same id, then the second request will be ignored and the // first [google.longrunning.Operation][google.longrunning.Operation] created and stored in the backend // is returned. // @@ -842,8 +1012,9 @@ message UpdateClusterRequest { google.protobuf.FieldMask update_mask = 4 [(google.api.field_behavior) = REQUIRED]; // Optional. A unique id used to identify the request. If the server - // receives two [UpdateClusterRequest][google.cloud.dataproc.v1.UpdateClusterRequest] requests with the same - // id, then the second request will be ignored and the + // receives two + // [UpdateClusterRequest](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#google.cloud.dataproc.v1.UpdateClusterRequest)s + // with the same id, then the second request will be ignored and the // first [google.longrunning.Operation][google.longrunning.Operation] created and stored in the // backend is returned. // @@ -855,6 +1026,68 @@ message UpdateClusterRequest { string request_id = 7 [(google.api.field_behavior) = OPTIONAL]; } +// A request to stop a cluster. +message StopClusterRequest { + // Required. The ID of the Google Cloud Platform project the + // cluster belongs to. + string project_id = 1 [(google.api.field_behavior) = REQUIRED]; + + // Required. The Dataproc region in which to handle the request. + string region = 2 [(google.api.field_behavior) = REQUIRED]; + + // Required. The cluster name. + string cluster_name = 3 [(google.api.field_behavior) = REQUIRED]; + + // Optional. Specifying the `cluster_uuid` means the RPC will fail + // (with error NOT_FOUND) if a cluster with the specified UUID does not exist. + string cluster_uuid = 4 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. A unique id used to identify the request. If the server + // receives two + // [StopClusterRequest](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#google.cloud.dataproc.v1.StopClusterRequest)s + // with the same id, then the second request will be ignored and the + // first [google.longrunning.Operation][google.longrunning.Operation] created and stored in the + // backend is returned. + // + // Recommendation: Set this value to a + // [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier). + // + // The id must contain only letters (a-z, A-Z), numbers (0-9), + // underscores (_), and hyphens (-). The maximum length is 40 characters. + string request_id = 5 [(google.api.field_behavior) = OPTIONAL]; +} + +// A request to start a cluster. +message StartClusterRequest { + // Required. The ID of the Google Cloud Platform project the + // cluster belongs to. + string project_id = 1 [(google.api.field_behavior) = REQUIRED]; + + // Required. The Dataproc region in which to handle the request. + string region = 2 [(google.api.field_behavior) = REQUIRED]; + + // Required. The cluster name. + string cluster_name = 3 [(google.api.field_behavior) = REQUIRED]; + + // Optional. Specifying the `cluster_uuid` means the RPC will fail + // (with error NOT_FOUND) if a cluster with the specified UUID does not exist. + string cluster_uuid = 4 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. A unique id used to identify the request. If the server + // receives two + // [StartClusterRequest](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#google.cloud.dataproc.v1.StartClusterRequest)s + // with the same id, then the second request will be ignored and the + // first [google.longrunning.Operation][google.longrunning.Operation] created and stored in the + // backend is returned. + // + // Recommendation: Set this value to a + // [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier). + // + // The id must contain only letters (a-z, A-Z), numbers (0-9), + // underscores (_), and hyphens (-). The maximum length is 40 characters. + string request_id = 5 [(google.api.field_behavior) = OPTIONAL]; +} + // A request to delete a cluster. message DeleteClusterRequest { // Required. The ID of the Google Cloud Platform project that the cluster @@ -872,8 +1105,9 @@ message DeleteClusterRequest { string cluster_uuid = 4 [(google.api.field_behavior) = OPTIONAL]; // Optional. A unique id used to identify the request. If the server - // receives two [DeleteClusterRequest][google.cloud.dataproc.v1.DeleteClusterRequest] requests with the same - // id, then the second request will be ignored and the + // receives two + // [DeleteClusterRequest](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#google.cloud.dataproc.v1.DeleteClusterRequest)s + // with the same id, then the second request will be ignored and the // first [google.longrunning.Operation][google.longrunning.Operation] created and stored in the // backend is returned. // diff --git a/google/cloud/dataproc/v1/dataproc_v1.yaml b/google/cloud/dataproc/v1/dataproc_v1.yaml index c6bdfc5fbf..fa01e726d7 100644 --- a/google/cloud/dataproc/v1/dataproc_v1.yaml +++ b/google/cloud/dataproc/v1/dataproc_v1.yaml @@ -1,5 +1,5 @@ type: google.api.Service -config_version: 2 +config_version: 3 name: dataproc.googleapis.com title: Cloud Dataproc API diff --git a/google/cloud/dataproc/v1/jobs.proto b/google/cloud/dataproc/v1/jobs.proto index 065530f35f..f3521b5db6 100644 --- a/google/cloud/dataproc/v1/jobs.proto +++ b/google/cloud/dataproc/v1/jobs.proto @@ -32,7 +32,8 @@ option java_package = "com.google.cloud.dataproc.v1"; // The JobController provides methods to manage jobs. service JobController { option (google.api.default_host) = "dataproc.googleapis.com"; - option (google.api.oauth_scopes) = "https://www.googleapis.com/auth/cloud-platform"; + option (google.api.oauth_scopes) = + "https://www.googleapis.com/auth/cloud-platform"; // Submits a job to a cluster. rpc SubmitJob(SubmitJobRequest) returns (Job) { @@ -44,7 +45,8 @@ service JobController { } // Submits job to a cluster. - rpc SubmitJobAsOperation(SubmitJobRequest) returns (google.longrunning.Operation) { + rpc SubmitJobAsOperation(SubmitJobRequest) + returns (google.longrunning.Operation) { option (google.api.http) = { post: "/v1/projects/{project_id}/regions/{region}/jobs:submitAsOperation" body: "*" @@ -286,9 +288,9 @@ message PySparkJob { // A list of queries to run on a cluster. message QueryList { - // Required. The queries to execute. You do not need to terminate a query - // with a semicolon. Multiple queries can be specified in one string - // by separating each with a semicolon. Here is an example of an Cloud + // Required. The queries to execute. You do not need to end a query expression + // with a semicolon. Multiple queries can be specified in one + // string by separating each with a semicolon. Here is an example of a // Dataproc API snippet that uses a QueryList to specify a HiveJob: // // "hiveJob": { @@ -323,7 +325,8 @@ message HiveJob { // Optional. Mapping of query variable names to values (equivalent to the // Hive command: `SET name="value";`). - map script_variables = 4 [(google.api.field_behavior) = OPTIONAL]; + map script_variables = 4 + [(google.api.field_behavior) = OPTIONAL]; // Optional. A mapping of property names and values, used to configure Hive. // Properties that conflict with values set by the Dataproc API may be @@ -352,7 +355,8 @@ message SparkSqlJob { // Optional. Mapping of query variable names to values (equivalent to the // Spark SQL command: SET `name="value";`). - map script_variables = 3 [(google.api.field_behavior) = OPTIONAL]; + map script_variables = 3 + [(google.api.field_behavior) = OPTIONAL]; // Optional. A mapping of property names to values, used to configure // Spark SQL's SparkConf. Properties that conflict with values set by the @@ -386,7 +390,8 @@ message PigJob { // Optional. Mapping of query variable names to values (equivalent to the Pig // command: `name=[value]`). - map script_variables = 4 [(google.api.field_behavior) = OPTIONAL]; + map script_variables = 4 + [(google.api.field_behavior) = OPTIONAL]; // Optional. A mapping of property names to values, used to configure Pig. // Properties that conflict with values set by the Dataproc API may be @@ -479,6 +484,11 @@ message JobPlacement { // Output only. A cluster UUID generated by the Dataproc service when // the job is submitted. string cluster_uuid = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Optional. Cluster labels to identify a cluster where the job will be + // submitted. + map cluster_labels = 3 + [(google.api.field_behavior) = OPTIONAL]; } // Dataproc job status. @@ -557,7 +567,8 @@ message JobStatus { ]; // Output only. The time when this state was entered. - google.protobuf.Timestamp state_start_time = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; + google.protobuf.Timestamp state_start_time = 6 + [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Additional state information, which includes // status reported by the agent. @@ -566,8 +577,8 @@ message JobStatus { // Encapsulates the full scoping used to reference a job. message JobReference { - // Optional. The ID of the Google Cloud Platform project that the job belongs to. If - // specified, must match the request project ID. + // Optional. The ID of the Google Cloud Platform project that the job belongs + // to. If specified, must match the request project ID. string project_id = 1 [(google.api.field_behavior) = OPTIONAL]; // Optional. The job ID, which must be unique within the project. @@ -677,22 +688,26 @@ message Job { JobStatus status = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The previous job status. - repeated JobStatus status_history = 13 [(google.api.field_behavior) = OUTPUT_ONLY]; + repeated JobStatus status_history = 13 + [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The collection of YARN applications spun up by this job. // // **Beta** Feature: This report is available for testing purposes only. It // may be changed before final release. - repeated YarnApplication yarn_applications = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; + repeated YarnApplication yarn_applications = 9 + [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. A URI pointing to the location of the stdout of the job's // driver program. - string driver_output_resource_uri = 17 [(google.api.field_behavior) = OUTPUT_ONLY]; + string driver_output_resource_uri = 17 + [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. If present, the location of miscellaneous control files // which may be used as part of job setup and handling. If not present, // control files may be placed in the same location as `driver_output_uri`. - string driver_control_files_uri = 15 [(google.api.field_behavior) = OUTPUT_ONLY]; + string driver_control_files_uri = 15 + [(google.api.field_behavior) = OUTPUT_ONLY]; // Optional. The labels to associate with this job. // Label **keys** must contain 1 to 63 characters, and must conform to @@ -711,8 +726,8 @@ message Job { // may be reused over time. string job_uuid = 22 [(google.api.field_behavior) = OUTPUT_ONLY]; - // Output only. Indicates whether the job is completed. If the value is `false`, - // the job is still in progress. If `true`, the job is completed, and + // Output only. Indicates whether the job is completed. If the value is + // `false`, the job is still in progress. If `true`, the job is completed, and // `status.state` field will indicate if it was successful, failed, // or cancelled. bool done = 24 [(google.api.field_behavior) = OUTPUT_ONLY]; @@ -721,7 +736,7 @@ message Job { // Job scheduling options. message JobScheduling { // Optional. Maximum number of times per hour a driver may be restarted as - // a result of driver terminating with non-zero code before job is + // a result of driver exiting with non-zero code before job is // reported failed. // // A job may be reported as thrashing if driver exits with non-zero code @@ -729,6 +744,11 @@ message JobScheduling { // // Maximum value is 10. int32 max_failures_per_hour = 1 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Maximum number of times in total a driver may be restarted as a + // result of driver exiting with non-zero code before job is reported failed. + // Maximum value is 240. + int32 max_failures_total = 2 [(google.api.field_behavior) = OPTIONAL]; } // A request to submit a job. @@ -744,8 +764,9 @@ message SubmitJobRequest { Job job = 2 [(google.api.field_behavior) = REQUIRED]; // Optional. A unique id used to identify the request. If the server - // receives two [SubmitJobRequest][google.cloud.dataproc.v1.SubmitJobRequest] requests with the same - // id, then the second request will be ignored and the + // receives two + // [SubmitJobRequest](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#google.cloud.dataproc.v1.SubmitJobRequest)s + // with the same id, then the second request will be ignored and the // first [Job][google.cloud.dataproc.v1.Job] created and stored in the backend // is returned. // @@ -769,7 +790,8 @@ message JobMetadata { string operation_type = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Job submission time. - google.protobuf.Timestamp start_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; + google.protobuf.Timestamp start_time = 4 + [(google.api.field_behavior) = OUTPUT_ONLY]; } // A request to get the resource representation for a job in a project. @@ -822,7 +844,8 @@ message ListJobsRequest { // (default = match ALL jobs). // // If `filter` is provided, `jobStateMatcher` will be ignored. - JobStateMatcher job_state_matcher = 5 [(google.api.field_behavior) = OPTIONAL]; + JobStateMatcher job_state_matcher = 5 + [(google.api.field_behavior) = OPTIONAL]; // Optional. A filter constraining the jobs to list. Filters are // case-sensitive and have the following syntax: @@ -862,7 +885,8 @@ message UpdateJobRequest { // labels, and the `PATCH` request body would specify the new // value. Note: Currently, labels is the only // field that can be updated. - google.protobuf.FieldMask update_mask = 5 [(google.api.field_behavior) = REQUIRED]; + google.protobuf.FieldMask update_mask = 5 + [(google.api.field_behavior) = REQUIRED]; } // A list of jobs in a project. diff --git a/google/cloud/dataproc/v1/shared.proto b/google/cloud/dataproc/v1/shared.proto index 7a1382f1dd..176e4535cf 100644 --- a/google/cloud/dataproc/v1/shared.proto +++ b/google/cloud/dataproc/v1/shared.proto @@ -17,6 +17,7 @@ syntax = "proto3"; package google.cloud.dataproc.v1; import "google/api/annotations.proto"; +import "google/api/field_behavior.proto"; option go_package = "google.golang.org/genproto/googleapis/cloud/dataproc/v1;dataproc"; option java_multiple_files = true; @@ -24,13 +25,30 @@ option java_outer_classname = "SharedProto"; option java_package = "com.google.cloud.dataproc.v1"; // Cluster components that can be activated. +// Next ID: 16. enum Component { // Unspecified component. Specifying this will cause Cluster creation to fail. COMPONENT_UNSPECIFIED = 0; - // The Anaconda python distribution. + // The Anaconda python distribution. The Anaconda component is not supported + // in the Dataproc + // 2.0 + // image. The 2.0 image is pre-installed with Miniconda. ANACONDA = 5; + // Docker + DOCKER = 13; + + // The Druid query engine. (alpha) + DRUID = 9; + + // Flink + FLINK = 14; + + // HBase. (beta) + HBASE = 11; + // The Hive Web HCatalog (the REST service for accessing HCatalog). HIVE_WEBHCAT = 3; @@ -40,6 +58,12 @@ enum Component { // The Presto query engine. PRESTO = 6; + // The Ranger service. + RANGER = 12; + + // The Solr service. + SOLR = 10; + // The Zeppelin notebook. ZEPPELIN = 4; diff --git a/google/cloud/dataproc/v1/workflow_templates.proto b/google/cloud/dataproc/v1/workflow_templates.proto index 04f8100470..ea5bfd05eb 100644 --- a/google/cloud/dataproc/v1/workflow_templates.proto +++ b/google/cloud/dataproc/v1/workflow_templates.proto @@ -23,6 +23,7 @@ import "google/api/resource.proto"; import "google/cloud/dataproc/v1/clusters.proto"; import "google/cloud/dataproc/v1/jobs.proto"; import "google/longrunning/operations.proto"; +import "google/protobuf/duration.proto"; import "google/protobuf/empty.proto"; import "google/protobuf/timestamp.proto"; @@ -35,10 +36,12 @@ option java_package = "com.google.cloud.dataproc.v1"; // Dataproc API. service WorkflowTemplateService { option (google.api.default_host) = "dataproc.googleapis.com"; - option (google.api.oauth_scopes) = "https://www.googleapis.com/auth/cloud-platform"; + option (google.api.oauth_scopes) = + "https://www.googleapis.com/auth/cloud-platform"; // Creates new workflow template. - rpc CreateWorkflowTemplate(CreateWorkflowTemplateRequest) returns (WorkflowTemplate) { + rpc CreateWorkflowTemplate(CreateWorkflowTemplateRequest) + returns (WorkflowTemplate) { option (google.api.http) = { post: "/v1/{parent=projects/*/locations/*}/workflowTemplates" body: "template" @@ -54,7 +57,8 @@ service WorkflowTemplateService { // // Can retrieve previously instantiated template by specifying optional // version parameter. - rpc GetWorkflowTemplate(GetWorkflowTemplateRequest) returns (WorkflowTemplate) { + rpc GetWorkflowTemplate(GetWorkflowTemplateRequest) + returns (WorkflowTemplate) { option (google.api.http) = { get: "/v1/{name=projects/*/locations/*/workflowTemplates/*}" additional_bindings { @@ -84,7 +88,8 @@ service WorkflowTemplateService { // On successful completion, // [Operation.response][google.longrunning.Operation.response] will be // [Empty][google.protobuf.Empty]. - rpc InstantiateWorkflowTemplate(InstantiateWorkflowTemplateRequest) returns (google.longrunning.Operation) { + rpc InstantiateWorkflowTemplate(InstantiateWorkflowTemplateRequest) + returns (google.longrunning.Operation) { option (google.api.http) = { post: "/v1/{name=projects/*/locations/*/workflowTemplates/*}:instantiate" body: "*" @@ -104,7 +109,8 @@ service WorkflowTemplateService { // Instantiates a template and begins execution. // // This method is equivalent to executing the sequence - // [CreateWorkflowTemplate][google.cloud.dataproc.v1.WorkflowTemplateService.CreateWorkflowTemplate], [InstantiateWorkflowTemplate][google.cloud.dataproc.v1.WorkflowTemplateService.InstantiateWorkflowTemplate], + // [CreateWorkflowTemplate][google.cloud.dataproc.v1.WorkflowTemplateService.CreateWorkflowTemplate], + // [InstantiateWorkflowTemplate][google.cloud.dataproc.v1.WorkflowTemplateService.InstantiateWorkflowTemplate], // [DeleteWorkflowTemplate][google.cloud.dataproc.v1.WorkflowTemplateService.DeleteWorkflowTemplate]. // // The returned Operation can be used to track execution of @@ -125,7 +131,9 @@ service WorkflowTemplateService { // On successful completion, // [Operation.response][google.longrunning.Operation.response] will be // [Empty][google.protobuf.Empty]. - rpc InstantiateInlineWorkflowTemplate(InstantiateInlineWorkflowTemplateRequest) returns (google.longrunning.Operation) { + rpc InstantiateInlineWorkflowTemplate( + InstantiateInlineWorkflowTemplateRequest) + returns (google.longrunning.Operation) { option (google.api.http) = { post: "/v1/{parent=projects/*/locations/*}/workflowTemplates:instantiateInline" body: "template" @@ -143,7 +151,8 @@ service WorkflowTemplateService { // Updates (replaces) workflow template. The updated template // must contain version that matches the current server version. - rpc UpdateWorkflowTemplate(UpdateWorkflowTemplateRequest) returns (WorkflowTemplate) { + rpc UpdateWorkflowTemplate(UpdateWorkflowTemplateRequest) + returns (WorkflowTemplate) { option (google.api.http) = { put: "/v1/{template.name=projects/*/locations/*/workflowTemplates/*}" body: "template" @@ -156,7 +165,8 @@ service WorkflowTemplateService { } // Lists workflows that match the specified filter in the request. - rpc ListWorkflowTemplates(ListWorkflowTemplatesRequest) returns (ListWorkflowTemplatesResponse) { + rpc ListWorkflowTemplates(ListWorkflowTemplatesRequest) + returns (ListWorkflowTemplatesResponse) { option (google.api.http) = { get: "/v1/{parent=projects/*/locations/*}/workflowTemplates" additional_bindings { @@ -167,7 +177,8 @@ service WorkflowTemplateService { } // Deletes a workflow template. It does not cancel in-progress workflows. - rpc DeleteWorkflowTemplate(DeleteWorkflowTemplateRequest) returns (google.protobuf.Empty) { + rpc DeleteWorkflowTemplate(DeleteWorkflowTemplateRequest) + returns (google.protobuf.Empty) { option (google.api.http) = { delete: "/v1/{name=projects/*/locations/*/workflowTemplates/*}" additional_bindings { @@ -213,10 +224,12 @@ message WorkflowTemplate { int32 version = 3 [(google.api.field_behavior) = OPTIONAL]; // Output only. The time template was created. - google.protobuf.Timestamp create_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; + google.protobuf.Timestamp create_time = 4 + [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The time template was last updated. - google.protobuf.Timestamp update_time = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; + google.protobuf.Timestamp update_time = 5 + [(google.api.field_behavior) = OUTPUT_ONLY]; // Optional. The labels to associate with this template. These labels // will be propagated to all jobs and clusters created by the workflow @@ -233,7 +246,8 @@ message WorkflowTemplate { map labels = 6 [(google.api.field_behavior) = OPTIONAL]; // Required. WorkflowTemplate scheduling information. - WorkflowTemplatePlacement placement = 7 [(google.api.field_behavior) = REQUIRED]; + WorkflowTemplatePlacement placement = 7 + [(google.api.field_behavior) = REQUIRED]; // Required. The Directed Acyclic Graph of Jobs to submit. repeated OrderedJob jobs = 8 [(google.api.field_behavior) = REQUIRED]; @@ -241,7 +255,21 @@ message WorkflowTemplate { // Optional. Template parameters whose values are substituted into the // template. Values for parameters must be provided when the template is // instantiated. - repeated TemplateParameter parameters = 9 [(google.api.field_behavior) = OPTIONAL]; + repeated TemplateParameter parameters = 9 + [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Timeout duration for the DAG of jobs, expressed in seconds (see + // [JSON representation of + // duration](https://developers.google.com/protocol-buffers/docs/proto3#json)). + // The timeout duration must be from 10 minutes ("600s") to 24 hours + // ("86400s"). The timer begins when the first job is submitted. If the + // workflow is running at the end of the timeout period, any remaining jobs + // are cancelled, the workflow is ended, and if the workflow was running on a + // [managed + // cluster](/dataproc/docs/concepts/workflows/using-workflows#configuring_or_selecting_a_cluster), + // the cluster is deleted. + google.protobuf.Duration dag_timeout = 10 + [(google.api.field_behavior) = OPTIONAL]; } // Specifies workflow execution target. @@ -299,7 +327,8 @@ message ClusterSelector { // Required. The cluster labels. Cluster must have all labels // to match. - map cluster_labels = 2 [(google.api.field_behavior) = REQUIRED]; + map cluster_labels = 2 + [(google.api.field_behavior) = REQUIRED]; } // A job executed by the workflow. @@ -309,8 +338,8 @@ message OrderedJob { // // The step id is used as prefix for job id, as job // `goog-dataproc-workflow-step-id` label, and in - // [prerequisiteStepIds][google.cloud.dataproc.v1.OrderedJob.prerequisite_step_ids] field from other - // steps. + // [prerequisiteStepIds][google.cloud.dataproc.v1.OrderedJob.prerequisite_step_ids] + // field from other steps. // // The id must contain only letters (a-z, A-Z), numbers (0-9), // underscores (_), and hyphens (-). Cannot begin or end with underscore @@ -361,7 +390,8 @@ message OrderedJob { // Optional. The optional list of prerequisite job step_ids. // If not specified, the job will start at the beginning of workflow. - repeated string prerequisite_step_ids = 10 [(google.api.field_behavior) = OPTIONAL]; + repeated string prerequisite_step_ids = 10 + [(google.api.field_behavior) = OPTIONAL]; } // A configurable parameter that replaces one or more fields in the template. @@ -387,10 +417,10 @@ message TemplateParameter { // A field is allowed to appear in at most one parameter's list of field // paths. // - // A field path is similar in syntax to a [google.protobuf.FieldMask][google.protobuf.FieldMask]. - // For example, a field path that references the zone field of a workflow - // template's cluster selector would be specified as - // `placement.clusterSelector.zone`. + // A field path is similar in syntax to a + // [google.protobuf.FieldMask][google.protobuf.FieldMask]. For example, a + // field path that references the zone field of a workflow template's cluster + // selector would be specified as `placement.clusterSelector.zone`. // // Also, field paths can reference fields using the following syntax: // @@ -497,13 +527,15 @@ message WorkflowMetadata { int32 version = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The create cluster operation metadata. - ClusterOperation create_cluster = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; + ClusterOperation create_cluster = 3 + [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The workflow graph. WorkflowGraph graph = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The delete cluster operation metadata. - ClusterOperation delete_cluster = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; + ClusterOperation delete_cluster = 5 + [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The workflow state. State state = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; @@ -515,13 +547,33 @@ message WorkflowMetadata { map parameters = 8; // Output only. Workflow start time. - google.protobuf.Timestamp start_time = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; + google.protobuf.Timestamp start_time = 9 + [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Workflow end time. - google.protobuf.Timestamp end_time = 10 [(google.api.field_behavior) = OUTPUT_ONLY]; + google.protobuf.Timestamp end_time = 10 + [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The UUID of target cluster. string cluster_uuid = 11 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The timeout duration for the DAG of jobs, expressed in seconds + // (see [JSON representation of + // duration](https://developers.google.com/protocol-buffers/docs/proto3#json)). + google.protobuf.Duration dag_timeout = 12 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. DAG start time, only set for workflows with + // [dag_timeout][google.cloud.dataproc.v1.WorkflowMetadata.dag_timeout] when + // DAG begins. + google.protobuf.Timestamp dag_start_time = 13 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. DAG end time, only set for workflows with + // [dag_timeout][google.cloud.dataproc.v1.WorkflowMetadata.dag_timeout] when + // DAG ends. + google.protobuf.Timestamp dag_end_time = 14 + [(google.api.field_behavior) = OUTPUT_ONLY]; } // The cluster operation triggered by a workflow. @@ -570,7 +622,8 @@ message WorkflowNode { string step_id = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Node's prerequisite nodes. - repeated string prerequisite_step_ids = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; + repeated string prerequisite_step_ids = 2 + [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The job id; populated after the node enters RUNNING state. string job_id = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; @@ -670,7 +723,7 @@ message InstantiateWorkflowTemplateRequest { string request_id = 5 [(google.api.field_behavior) = OPTIONAL]; // Optional. Map from parameter names to values that should be used for those - // parameters. Values may not exceed 100 characters. + // parameters. Values may not exceed 1000 characters. map parameters = 6 [(google.api.field_behavior) = OPTIONAL]; } @@ -746,7 +799,8 @@ message ListWorkflowTemplatesRequest { // A response to a request to list workflow templates in a project. message ListWorkflowTemplatesResponse { // Output only. WorkflowTemplates list. - repeated WorkflowTemplate templates = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + repeated WorkflowTemplate templates = 1 + [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. This token is included in the response if there are more // results to fetch. To fetch additional results, provide this value as the