diff --git a/google/cloud/dataproc/v1/BUILD.bazel b/google/cloud/dataproc/v1/BUILD.bazel index 391aaf6f28..6f15914c5f 100644 --- a/google/cloud/dataproc/v1/BUILD.bazel +++ b/google/cloud/dataproc/v1/BUILD.bazel @@ -9,6 +9,7 @@ load("@com_google_googleapis_imports//:imports.bzl", "proto_library_with_info") proto_library( name = "dataproc_proto", srcs = [ + "autoscaling_policies.proto", "clusters.proto", "jobs.proto", "operations.proto", diff --git a/google/cloud/dataproc/v1/autoscaling_policies.proto b/google/cloud/dataproc/v1/autoscaling_policies.proto new file mode 100644 index 0000000000..cb466ee851 --- /dev/null +++ b/google/cloud/dataproc/v1/autoscaling_policies.proto @@ -0,0 +1,340 @@ +// Copyright 2019 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +package google.cloud.dataproc.v1; + +import "google/api/annotations.proto"; +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/empty.proto"; +import "google/api/client.proto"; + +option go_package = "google.golang.org/genproto/googleapis/cloud/dataproc/v1;dataproc"; +option java_multiple_files = true; +option java_outer_classname = "AutoscalingPoliciesProto"; +option java_package = "com.google.cloud.dataproc.v1"; + +// The API interface for managing autoscaling policies in the +// Google Cloud Dataproc API. +service AutoscalingPolicyService { + option (google.api.default_host) = "dataproc.googleapis.com"; + option (google.api.oauth_scopes) = "https://www.googleapis.com/auth/cloud-platform"; + + // Creates new autoscaling policy. + rpc CreateAutoscalingPolicy(CreateAutoscalingPolicyRequest) returns (AutoscalingPolicy) { + option (google.api.http) = { + post: "/v1/{parent=projects/*/locations/*}/autoscalingPolicies" + body: "policy" + additional_bindings { + post: "/v1/{parent=projects/*/regions/*}/autoscalingPolicies" + body: "policy" + } + }; + } + + // Updates (replaces) autoscaling policy. + // + // Disabled check for update_mask, because all updates will be full + // replacements. + rpc UpdateAutoscalingPolicy(UpdateAutoscalingPolicyRequest) returns (AutoscalingPolicy) { + option (google.api.http) = { + put: "/v1/{policy.name=projects/*/locations/*/autoscalingPolicies/*}" + body: "policy" + additional_bindings { + put: "/v1/{policy.name=projects/*/regions/*/autoscalingPolicies/*}" + body: "policy" + } + }; + } + + // Retrieves autoscaling policy. + rpc GetAutoscalingPolicy(GetAutoscalingPolicyRequest) returns (AutoscalingPolicy) { + option (google.api.http) = { + get: "/v1/{name=projects/*/locations/*/autoscalingPolicies/*}" + additional_bindings { + get: "/v1/{name=projects/*/regions/*/autoscalingPolicies/*}" + } + }; + } + + // Lists autoscaling policies in the project. + rpc ListAutoscalingPolicies(ListAutoscalingPoliciesRequest) returns (ListAutoscalingPoliciesResponse) { + option (google.api.http) = { + get: "/v1/{parent=projects/*/locations/*}/autoscalingPolicies" + additional_bindings { + get: "/v1/{parent=projects/*/regions/*}/autoscalingPolicies" + } + }; + } + + // Deletes an autoscaling policy. It is an error to delete an autoscaling + // policy that is in use by one or more clusters. + rpc DeleteAutoscalingPolicy(DeleteAutoscalingPolicyRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { + delete: "/v1/{name=projects/*/locations/*/autoscalingPolicies/*}" + additional_bindings { + delete: "/v1/{name=projects/*/regions/*/autoscalingPolicies/*}" + } + }; + } +} + +// Describes an autoscaling policy for Dataproc cluster autoscaler. +message AutoscalingPolicy { + option (google.api.resource) = { + type: "dataproc.googleapis.com/AutoscalingPolicy" + pattern: "projects/{project}/regions/{region}/autoscalingPolicies/{autoscaling_policy}" + }; + + // Required. The policy id. + // + // The id must contain only letters (a-z, A-Z), numbers (0-9), + // underscores (_), and hyphens (-). Cannot begin or end with underscore + // or hyphen. Must consist of between 3 and 50 characters. + // + string id = 1; + + // Output only. The "resource name" of the autoscaling policy, as described + // in https://cloud.google.com/apis/design/resource_names. + // + // * For `projects.regions.autoscalingPolicies`, the resource name of the + // policy has the following format: + // `projects/{project_id}/regions/{region}/autoscalingPolicies/{policy_id}` + // + // * For `projects.locations.autoscalingPolicies`, the resource name of the + // policy has the following format: + // `projects/{project_id}/locations/{location}/autoscalingPolicies/{policy_id}` + string name = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Autoscaling algorithm for policy. + oneof algorithm { + BasicAutoscalingAlgorithm basic_algorithm = 3 [(google.api.field_behavior) = REQUIRED]; + } + + // Required. Describes how the autoscaler will operate for primary workers. + InstanceGroupAutoscalingPolicyConfig worker_config = 4 [(google.api.field_behavior) = REQUIRED]; + + // Optional. Describes how the autoscaler will operate for secondary workers. + InstanceGroupAutoscalingPolicyConfig secondary_worker_config = 5 [(google.api.field_behavior) = OPTIONAL]; +} + +// Basic algorithm for autoscaling. +message BasicAutoscalingAlgorithm { + // Required. YARN autoscaling configuration. + BasicYarnAutoscalingConfig yarn_config = 1 [(google.api.field_behavior) = REQUIRED]; + + // Optional. Duration between scaling events. A scaling period starts after + // the update operation from the previous event has completed. + // + // Bounds: [2m, 1d]. Default: 2m. + google.protobuf.Duration cooldown_period = 2 [(google.api.field_behavior) = OPTIONAL]; +} + +// Basic autoscaling configurations for YARN. +message BasicYarnAutoscalingConfig { + // Required. Timeout for YARN graceful decommissioning of Node Managers. + // Specifies the duration to wait for jobs to complete before forcefully + // removing workers (and potentially interrupting jobs). Only applicable to + // downscaling operations. + // + // Bounds: [0s, 1d]. + google.protobuf.Duration graceful_decommission_timeout = 5 [(google.api.field_behavior) = REQUIRED]; + + // Required. Fraction of average pending memory in the last cooldown period + // for which to add workers. A scale-up factor of 1.0 will result in scaling + // up so that there is no pending memory remaining after the update (more + // aggressive scaling). A scale-up factor closer to 0 will result in a smaller + // magnitude of scaling up (less aggressive scaling). + // + // Bounds: [0.0, 1.0]. + double scale_up_factor = 1 [(google.api.field_behavior) = REQUIRED]; + + // Required. Fraction of average pending memory in the last cooldown period + // for which to remove workers. A scale-down factor of 1 will result in + // scaling down so that there is no available memory remaining after the + // update (more aggressive scaling). A scale-down factor of 0 disables + // removing workers, which can be beneficial for autoscaling a single job. + // + // Bounds: [0.0, 1.0]. + double scale_down_factor = 2 [(google.api.field_behavior) = REQUIRED]; + + // Optional. Minimum scale-up threshold as a fraction of total cluster size + // before scaling occurs. For example, in a 20-worker cluster, a threshold of + // 0.1 means the autoscaler must recommend at least a 2-worker scale-up for + // the cluster to scale. A threshold of 0 means the autoscaler will scale up + // on any recommended change. + // + // Bounds: [0.0, 1.0]. Default: 0.0. + double scale_up_min_worker_fraction = 3 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Minimum scale-down threshold as a fraction of total cluster size + // before scaling occurs. For example, in a 20-worker cluster, a threshold of + // 0.1 means the autoscaler must recommend at least a 2 worker scale-down for + // the cluster to scale. A threshold of 0 means the autoscaler will scale down + // on any recommended change. + // + // Bounds: [0.0, 1.0]. Default: 0.0. + double scale_down_min_worker_fraction = 4 [(google.api.field_behavior) = OPTIONAL]; +} + +// Configuration for the size bounds of an instance group, including its +// proportional size to other groups. +message InstanceGroupAutoscalingPolicyConfig { + // Optional. Minimum number of instances for this group. + // + // Primary workers - Bounds: [2, max_instances]. Default: 2. + // Secondary workers - Bounds: [0, max_instances]. Default: 0. + int32 min_instances = 1 [(google.api.field_behavior) = OPTIONAL]; + + // Required. Maximum number of instances for this group. Required for primary + // workers. Note that by default, clusters will not use secondary workers. + // Required for secondary workers if the minimum secondary instances is set. + // + // Primary workers - Bounds: [min_instances, ). + // Secondary workers - Bounds: [min_instances, ). Default: 0. + int32 max_instances = 2 [(google.api.field_behavior) = REQUIRED]; + + // Optional. Weight for the instance group, which is used to determine the + // fraction of total workers in the cluster from this instance group. + // For example, if primary workers have weight 2, and secondary workers have + // weight 1, the cluster will have approximately 2 primary workers for each + // secondary worker. + // + // The cluster may not reach the specified balance if constrained + // by min/max bounds or other autoscaling settings. For example, if + // `max_instances` for secondary workers is 0, then only primary workers will + // be added. The cluster can also be out of balance when created. + // + // If weight is not set on any instance group, the cluster will default to + // equal weight for all groups: the cluster will attempt to maintain an equal + // number of workers in each group within the configured size bounds for each + // group. If weight is set for one group only, the cluster will default to + // zero weight on the unset group. For example if weight is set only on + // primary workers, the cluster will use primary workers only and no + // secondary workers. + int32 weight = 3 [(google.api.field_behavior) = OPTIONAL]; +} + +// A request to create an autoscaling policy. +message CreateAutoscalingPolicyRequest { + // Required. The "resource name" of the region or location, as described + // in https://cloud.google.com/apis/design/resource_names. + // + // * For `projects.regions.autoscalingPolicies.create`, the resource name + // of the region has the following format: + // `projects/{project_id}/regions/{region}` + // + // * For `projects.locations.autoscalingPolicies.create`, the resource name + // of the location has the following format: + // `projects/{project_id}/locations/{location}` + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + child_type: "dataproc.googleapis.com/AutoscalingPolicy" + } + ]; + + // The autoscaling policy to create. + AutoscalingPolicy policy = 2; +} + +// A request to fetch an autoscaling policy. +message GetAutoscalingPolicyRequest { + // Required. The "resource name" of the autoscaling policy, as described + // in https://cloud.google.com/apis/design/resource_names. + // + // * For `projects.regions.autoscalingPolicies.get`, the resource name + // of the policy has the following format: + // `projects/{project_id}/regions/{region}/autoscalingPolicies/{policy_id}` + // + // * For `projects.locations.autoscalingPolicies.get`, the resource name + // of the policy has the following format: + // `projects/{project_id}/locations/{location}/autoscalingPolicies/{policy_id}` + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "dataproc.googleapis.com/AutoscalingPolicy" + } + ]; +} + +// A request to update an autoscaling policy. +message UpdateAutoscalingPolicyRequest { + // Required. The updated autoscaling policy. + AutoscalingPolicy policy = 1 [(google.api.field_behavior) = REQUIRED]; +} + +// A request to delete an autoscaling policy. +// +// Autoscaling policies in use by one or more clusters will not be deleted. +message DeleteAutoscalingPolicyRequest { + // Required. The "resource name" of the autoscaling policy, as described + // in https://cloud.google.com/apis/design/resource_names. + // + // * For `projects.regions.autoscalingPolicies.delete`, the resource name + // of the policy has the following format: + // `projects/{project_id}/regions/{region}/autoscalingPolicies/{policy_id}` + // + // * For `projects.locations.autoscalingPolicies.delete`, the resource name + // of the policy has the following format: + // `projects/{project_id}/locations/{location}/autoscalingPolicies/{policy_id}` + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "dataproc.googleapis.com/AutoscalingPolicy" + } + ]; +} + +// A request to list autoscaling policies in a project. +message ListAutoscalingPoliciesRequest { + // Required. The "resource name" of the region or location, as described + // in https://cloud.google.com/apis/design/resource_names. + // + // * For `projects.regions.autoscalingPolicies.list`, the resource name + // of the region has the following format: + // `projects/{project_id}/regions/{region}` + // + // * For `projects.locations.autoscalingPolicies.list`, the resource name + // of the location has the following format: + // `projects/{project_id}/locations/{location}` + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + child_type: "dataproc.googleapis.com/AutoscalingPolicy" + } + ]; + + // Optional. The maximum number of results to return in each response. + // Must be less than or equal to 1000. Defaults to 100. + int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The page token, returned by a previous call, to request the + // next page of results. + string page_token = 3 [(google.api.field_behavior) = OPTIONAL]; +} + +// A response to a request to list autoscaling policies in a project. +message ListAutoscalingPoliciesResponse { + // Output only. Autoscaling policies list. + repeated AutoscalingPolicy policies = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. This token is included in the response if there are more + // results to fetch. + string next_page_token = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; +} diff --git a/google/cloud/dataproc/v1/clusters.proto b/google/cloud/dataproc/v1/clusters.proto index ab2cc2d025..c92abbdb52 100644 --- a/google/cloud/dataproc/v1/clusters.proto +++ b/google/cloud/dataproc/v1/clusters.proto @@ -18,7 +18,8 @@ syntax = "proto3"; package google.cloud.dataproc.v1; import "google/api/annotations.proto"; -import "google/cloud/dataproc/v1/operations.proto"; +import "google/api/client.proto"; +import "google/api/field_behavior.proto"; import "google/cloud/dataproc/v1/shared.proto"; import "google/longrunning/operations.proto"; import "google/protobuf/duration.proto"; @@ -33,27 +34,50 @@ option java_package = "com.google.cloud.dataproc.v1"; // The ClusterControllerService provides methods to manage clusters // of Compute Engine instances. service ClusterController { - // Creates a cluster in a project. + option (google.api.default_host) = "dataproc.googleapis.com"; + option (google.api.oauth_scopes) = "https://www.googleapis.com/auth/cloud-platform"; + + // Creates a cluster in a project. The returned + // [Operation.metadata][google.longrunning.Operation.metadata] will be + // [ClusterOperationMetadata](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata). rpc CreateCluster(CreateClusterRequest) returns (google.longrunning.Operation) { option (google.api.http) = { post: "/v1/projects/{project_id}/regions/{region}/clusters" body: "cluster" }; + option (google.api.method_signature) = "project_id,region,cluster"; + option (google.longrunning.operation_info) = { + response_type: "Cluster" + metadata_type: "google.cloud.dataproc.v1.ClusterOperationMetadata" + }; } - // Updates a cluster in a project. + // Updates a cluster in a project. The returned + // [Operation.metadata][google.longrunning.Operation.metadata] will be + // [ClusterOperationMetadata](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata). rpc UpdateCluster(UpdateClusterRequest) returns (google.longrunning.Operation) { option (google.api.http) = { patch: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}" body: "cluster" }; + option (google.longrunning.operation_info) = { + response_type: "Cluster" + metadata_type: "google.cloud.dataproc.v1.ClusterOperationMetadata" + }; } - // Deletes a cluster in a project. + // Deletes a cluster in a project. The returned + // [Operation.metadata][google.longrunning.Operation.metadata] will be + // [ClusterOperationMetadata](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata). rpc DeleteCluster(DeleteClusterRequest) returns (google.longrunning.Operation) { option (google.api.http) = { delete: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}" }; + option (google.api.method_signature) = "project_id,region,cluster_name"; + option (google.longrunning.operation_info) = { + response_type: "google.protobuf.Empty" + metadata_type: "google.cloud.dataproc.v1.ClusterOperationMetadata" + }; } // Gets the resource representation for a cluster in a project. @@ -61,6 +85,7 @@ service ClusterController { option (google.api.http) = { get: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}" }; + option (google.api.method_signature) = "project_id,region,cluster_name"; } // Lists all regions/{region}/clusters in a project. @@ -68,16 +93,27 @@ service ClusterController { option (google.api.http) = { get: "/v1/projects/{project_id}/regions/{region}/clusters" }; + option (google.api.method_signature) = "project_id,region"; + option (google.api.method_signature) = "project_id,region,filter"; } - // Gets cluster diagnostic information. - // After the operation completes, the Operation.response field - // contains `DiagnoseClusterOutputLocation`. + // Gets cluster diagnostic information. The returned + // [Operation.metadata][google.longrunning.Operation.metadata] will be + // [ClusterOperationMetadata](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata). + // After the operation completes, + // [Operation.response][google.longrunning.Operation.response] + // contains + // [DiagnoseClusterResults](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#diagnoseclusterresults). rpc DiagnoseCluster(DiagnoseClusterRequest) returns (google.longrunning.Operation) { option (google.api.http) = { post: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}:diagnose" body: "*" }; + option (google.api.method_signature) = "project_id,region,cluster_name"; + option (google.longrunning.operation_info) = { + response_type: "google.protobuf.Empty" + metadata_type: "DiagnoseClusterResults" + }; } } @@ -102,17 +138,17 @@ message Cluster { // characters, and must conform to [RFC // 1035](https://www.ietf.org/rfc/rfc1035.txt). No more than 32 labels can be // associated with a cluster. - map labels = 8; + map labels = 8 [(google.api.field_behavior) = OPTIONAL]; // Output only. Cluster status. - ClusterStatus status = 4; + ClusterStatus status = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The previous cluster status. - repeated ClusterStatus status_history = 7; + repeated ClusterStatus status_history = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. A cluster UUID (Unique Universal Identifier). Cloud Dataproc // generates this value when it creates the cluster. - string cluster_uuid = 6; + string cluster_uuid = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; // Contains cluster daemon metrics such as HDFS and YARN stats. // @@ -132,26 +168,26 @@ message ClusterConfig { // and manage this project-level, per-location bucket (see // [Cloud Dataproc staging // bucket](/dataproc/docs/concepts/configuring-clusters/staging-bucket)). - string config_bucket = 1; + string config_bucket = 1 [(google.api.field_behavior) = OPTIONAL]; // Optional. The shared Compute Engine config settings for // all instances in a cluster. - GceClusterConfig gce_cluster_config = 8; + GceClusterConfig gce_cluster_config = 8 [(google.api.field_behavior) = OPTIONAL]; // Optional. The Compute Engine config settings for // the master instance in a cluster. - InstanceGroupConfig master_config = 9; + InstanceGroupConfig master_config = 9 [(google.api.field_behavior) = OPTIONAL]; // Optional. The Compute Engine config settings for // worker instances in a cluster. - InstanceGroupConfig worker_config = 10; + InstanceGroupConfig worker_config = 10 [(google.api.field_behavior) = OPTIONAL]; // Optional. The Compute Engine config settings for // additional worker instances in a cluster. - InstanceGroupConfig secondary_worker_config = 12; + InstanceGroupConfig secondary_worker_config = 12 [(google.api.field_behavior) = OPTIONAL]; // Optional. The config settings for software inside the cluster. - SoftwareConfig software_config = 13; + SoftwareConfig software_config = 13 [(google.api.field_behavior) = OPTIONAL]; // Optional. Commands to execute on each node after config is // completed. By default, executables are run on master and all worker nodes. @@ -166,17 +202,38 @@ message ClusterConfig { // else // ... worker specific actions ... // fi - repeated NodeInitializationAction initialization_actions = 11; + repeated NodeInitializationAction initialization_actions = 11 [(google.api.field_behavior) = OPTIONAL]; // Optional. Encryption settings for the cluster. - EncryptionConfig encryption_config = 15; + EncryptionConfig encryption_config = 15 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Autoscaling config for the policy associated with the cluster. + // Cluster does not autoscale if this field is unset. + AutoscalingConfig autoscaling_config = 18 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Security settings for the cluster. + SecurityConfig security_config = 16 [(google.api.field_behavior) = OPTIONAL]; +} + +// Autoscaling Policy config associated with the cluster. +message AutoscalingConfig { + // Optional. The autoscaling policy used by the cluster. + // + // Only resource names including projectid and location (region) are valid. + // Examples: + // + // * `https://www.googleapis.com/compute/v1/projects/[project_id]/locations/[dataproc_region]/autoscalingPolicies/[policy_id]` + // * `projects/[project_id]/locations/[dataproc_region]/autoscalingPolicies/[policy_id]` + // + // Note that the policy must be in the same project and Cloud Dataproc region. + string policy_uri = 1 [(google.api.field_behavior) = OPTIONAL]; } // Encryption settings for the cluster. message EncryptionConfig { // Optional. The Cloud KMS key name to use for PD disk encryption for all // instances in the cluster. - string gce_pd_kms_key_name = 1; + string gce_pd_kms_key_name = 1 [(google.api.field_behavior) = OPTIONAL]; } // Common config settings for resources of Compute Engine cluster @@ -193,7 +250,7 @@ message GceClusterConfig { // * `https://www.googleapis.com/compute/v1/projects/[project_id]/zones/[zone]` // * `projects/[project_id]/zones/[zone]` // * `us-central1-f` - string zone_uri = 1; + string zone_uri = 1 [(google.api.field_behavior) = OPTIONAL]; // Optional. The Compute Engine network to be used for machine // communications. Cannot be specified with subnetwork_uri. If neither @@ -206,7 +263,7 @@ message GceClusterConfig { // * `https://www.googleapis.com/compute/v1/projects/[project_id]/regions/global/default` // * `projects/[project_id]/regions/global/default` // * `default` - string network_uri = 2; + string network_uri = 2 [(google.api.field_behavior) = OPTIONAL]; // Optional. The Compute Engine subnetwork to be used for machine // communications. Cannot be specified with network_uri. @@ -216,7 +273,7 @@ message GceClusterConfig { // * `https://www.googleapis.com/compute/v1/projects/[project_id]/regions/us-east1/subnetworks/sub0` // * `projects/[project_id]/regions/us-east1/subnetworks/sub0` // * `sub0` - string subnetwork_uri = 6; + string subnetwork_uri = 6 [(google.api.field_behavior) = OPTIONAL]; // Optional. If true, all instances in the cluster will only have internal IP // addresses. By default, clusters are not restricted to internal IP @@ -224,7 +281,7 @@ message GceClusterConfig { // instance. This `internal_ip_only` restriction can only be enabled for // subnetwork enabled networks, and all off-cluster dependencies must be // configured to be accessible without external IP addresses. - bool internal_ip_only = 7; + bool internal_ip_only = 7 [(google.api.field_behavior) = OPTIONAL]; // Optional. The service account of the instances. Defaults to the default // Compute Engine service account. Custom service accounts need @@ -237,7 +294,7 @@ message GceClusterConfig { // https://cloud.google.com/compute/docs/access/service-accounts#custom_service_accounts // for more information). // Example: `[account_id]@[project_id].iam.gserviceaccount.com` - string service_account = 8; + string service_account = 8 [(google.api.field_behavior) = OPTIONAL]; // Optional. The URIs of service account scopes to be included in // Compute Engine instances. The following base set of scopes is always @@ -253,7 +310,7 @@ message GceClusterConfig { // * https://www.googleapis.com/auth/bigtable.admin.table // * https://www.googleapis.com/auth/bigtable.data // * https://www.googleapis.com/auth/devstorage.full_control - repeated string service_account_scopes = 3; + repeated string service_account_scopes = 3 [(google.api.field_behavior) = OPTIONAL]; // The Compute Engine tags to add to all instances (see // [Tagging instances](/compute/docs/label-or-tag-resources#tags)). @@ -270,16 +327,16 @@ message GceClusterConfig { message InstanceGroupConfig { // Optional. The number of VM instances in the instance group. // For master instance groups, must be set to 1. - int32 num_instances = 1; + int32 num_instances = 1 [(google.api.field_behavior) = OPTIONAL]; // Output only. The list of instance names. Cloud Dataproc derives the names // from `cluster_name`, `num_instances`, and the instance group. - repeated string instance_names = 2; + repeated string instance_names = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Optional. The Compute Engine image resource used for cluster // instances. It can be specified or may be inferred from // `SoftwareConfig.image_version`. - string image_uri = 3; + string image_uri = 3 [(google.api.field_behavior) = OPTIONAL]; // Optional. The Compute Engine machine type used for cluster instances. // @@ -294,36 +351,38 @@ message InstanceGroupConfig { // Placement](/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) // feature, you must use the short name of the machine type // resource, for example, `n1-standard-2`. - string machine_type_uri = 4; + string machine_type_uri = 4 [(google.api.field_behavior) = OPTIONAL]; // Optional. Disk option config settings. - DiskConfig disk_config = 5; + DiskConfig disk_config = 5 [(google.api.field_behavior) = OPTIONAL]; // Optional. Specifies that this instance group contains preemptible // instances. - bool is_preemptible = 6; + bool is_preemptible = 6 [(google.api.field_behavior) = OPTIONAL]; // Output only. The config for Compute Engine Instance Group // Manager that manages this group. // This is only used for preemptible instance groups. - ManagedGroupConfig managed_group_config = 7; + ManagedGroupConfig managed_group_config = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; // Optional. The Compute Engine accelerator configuration for these // instances. - // - // **Beta Feature**: This feature is still under development. It may be - // changed before final release. - repeated AcceleratorConfig accelerators = 8; + repeated AcceleratorConfig accelerators = 8 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Specifies the minimum cpu platform for the Instance Group. + // See [Cloud Dataproc→Minimum CPU Platform] + // (/dataproc/docs/concepts/compute/dataproc-min-cpu). + string min_cpu_platform = 9 [(google.api.field_behavior) = OPTIONAL]; } // Specifies the resources used to actively manage an instance group. message ManagedGroupConfig { // Output only. The name of the Instance Template used for the Managed // Instance Group. - string instance_template_name = 1; + string instance_template_name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The name of the Instance Group Manager for this group. - string instance_group_manager_name = 2; + string instance_group_manager_name = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Specifies the type and number of accelerator cards attached to the instances @@ -356,10 +415,10 @@ message DiskConfig { // Optional. Type of the boot disk (default is "pd-standard"). // Valid values: "pd-ssd" (Persistent Disk Solid State Drive) or // "pd-standard" (Persistent Disk Hard Disk Drive). - string boot_disk_type = 3; + string boot_disk_type = 3 [(google.api.field_behavior) = OPTIONAL]; // Optional. Size in GB of the boot disk (default is 500GB). - int32 boot_disk_size_gb = 1; + int32 boot_disk_size_gb = 1 [(google.api.field_behavior) = OPTIONAL]; // Optional. Number of attached SSDs, from 0 to 4 (default is 0). // If SSDs are not attached, the boot disk is used to store runtime logs and @@ -367,20 +426,20 @@ message DiskConfig { // If one or more SSDs are attached, this runtime bulk // data is spread across them, and the boot disk contains only basic // config and installed binaries. - int32 num_local_ssds = 2; + int32 num_local_ssds = 2 [(google.api.field_behavior) = OPTIONAL]; } // Specifies an executable to run on a fully configured node and a // timeout period for executable completion. message NodeInitializationAction { // Required. Cloud Storage URI of executable file. - string executable_file = 1; + string executable_file = 1 [(google.api.field_behavior) = REQUIRED]; // Optional. Amount of time executable has to complete. Default is // 10 minutes. Cluster creation fails with an explanatory error message (the // name of the executable that caused the error and the exceeded timeout // period) if the executable is not completed at end of the timeout period. - google.protobuf.Duration execution_timeout = 2; + google.protobuf.Duration execution_timeout = 2 [(google.api.field_behavior) = OPTIONAL]; } // The status of a cluster and its instances. @@ -426,17 +485,95 @@ message ClusterStatus { } // Output only. The cluster's state. - State state = 1; + State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; - // Output only. Optional details of cluster's state. - string detail = 2; + // Optional. Output only. Details of cluster's state. + string detail = 2 [ + (google.api.field_behavior) = OUTPUT_ONLY, + (google.api.field_behavior) = OPTIONAL + ]; // Output only. Time when this state was entered. - google.protobuf.Timestamp state_start_time = 3; + google.protobuf.Timestamp state_start_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Additional state information that includes // status reported by the agent. - Substate substate = 4; + Substate substate = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// Security related configuration, including Kerberos. +message SecurityConfig { + // Kerberos related configuration. + KerberosConfig kerberos_config = 1; +} + +// Specifies Kerberos related configuration. +message KerberosConfig { + // Optional. Flag to indicate whether to Kerberize the cluster. + bool enable_kerberos = 1 [(google.api.field_behavior) = OPTIONAL]; + + // Required. The Cloud Storage URI of a KMS encrypted file containing the root + // principal password. + string root_principal_password_uri = 2 [(google.api.field_behavior) = REQUIRED]; + + // Required. The uri of the KMS key used to encrypt various sensitive + // files. + string kms_key_uri = 3 [(google.api.field_behavior) = REQUIRED]; + + // Optional. The Cloud Storage URI of the keystore file used for SSL + // encryption. If not provided, Dataproc will provide a self-signed + // certificate. + string keystore_uri = 4 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The Cloud Storage URI of the truststore file used for SSL + // encryption. If not provided, Dataproc will provide a self-signed + // certificate. + string truststore_uri = 5 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The Cloud Storage URI of a KMS encrypted file containing the + // password to the user provided keystore. For the self-signed certificate, + // this password is generated by Dataproc. + string keystore_password_uri = 6 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The Cloud Storage URI of a KMS encrypted file containing the + // password to the user provided key. For the self-signed certificate, this + // password is generated by Dataproc. + string key_password_uri = 7 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The Cloud Storage URI of a KMS encrypted file containing the + // password to the user provided truststore. For the self-signed certificate, + // this password is generated by Dataproc. + string truststore_password_uri = 8 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The remote realm the Dataproc on-cluster KDC will trust, should + // the user enable cross realm trust. + string cross_realm_trust_realm = 9 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The KDC (IP or hostname) for the remote trusted realm in a cross + // realm trust relationship. + string cross_realm_trust_kdc = 10 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The admin server (IP or hostname) for the remote trusted realm in + // a cross realm trust relationship. + string cross_realm_trust_admin_server = 11 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The Cloud Storage URI of a KMS encrypted file containing the + // shared password between the on-cluster Kerberos realm and the remote + // trusted realm, in a cross realm trust relationship. + string cross_realm_trust_shared_password_uri = 12 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The Cloud Storage URI of a KMS encrypted file containing the + // master key of the KDC database. + string kdc_db_key_uri = 13 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The lifetime of the ticket granting ticket, in hours. + // If not specified, or user specifies 0, then default value 10 + // will be used. + int32 tgt_lifetime_hours = 14 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The name of the on-cluster Kerberos realm. + // If not specified, the uppercased domain of hostnames will be the realm. + string realm = 15 [(google.api.field_behavior) = OPTIONAL]; } // Specifies the selection and config of software inside the cluster. @@ -448,7 +585,7 @@ message SoftwareConfig { // ["preview" // version](/dataproc/docs/concepts/versioning/dataproc-versions#other_versions). // If unspecified, it defaults to the latest Debian version. - string image_version = 1; + string image_version = 1 [(google.api.field_behavior) = OPTIONAL]; // Optional. The properties to set on daemon config files. // @@ -468,10 +605,10 @@ message SoftwareConfig { // // For more information, see // [Cluster properties](/dataproc/docs/concepts/cluster-properties). - map properties = 2; + map properties = 2 [(google.api.field_behavior) = OPTIONAL]; - // The set of optional components to activate on the cluster. - repeated Component optional_components = 3; + // Optional. The set of components to activate on the cluster. + repeated Component optional_components = 3 [(google.api.field_behavior) = OPTIONAL]; } // Contains cluster daemon metrics, such as HDFS and YARN stats. @@ -490,13 +627,13 @@ message ClusterMetrics { message CreateClusterRequest { // Required. The ID of the Google Cloud Platform project that the cluster // belongs to. - string project_id = 1; + string project_id = 1 [(google.api.field_behavior) = REQUIRED]; // Required. The Cloud Dataproc region in which to handle the request. - string region = 3; + string region = 3 [(google.api.field_behavior) = REQUIRED]; // Required. The cluster to create. - Cluster cluster = 2; + Cluster cluster = 2 [(google.api.field_behavior) = REQUIRED]; // Optional. A unique id used to identify the request. If the server // receives two [CreateClusterRequest][google.cloud.dataproc.v1.CreateClusterRequest] requests with the same @@ -509,23 +646,23 @@ message CreateClusterRequest { // // The id must contain only letters (a-z, A-Z), numbers (0-9), // underscores (_), and hyphens (-). The maximum length is 40 characters. - string request_id = 4; + string request_id = 4 [(google.api.field_behavior) = OPTIONAL]; } // A request to update a cluster. message UpdateClusterRequest { // Required. The ID of the Google Cloud Platform project the // cluster belongs to. - string project_id = 1; + string project_id = 1 [(google.api.field_behavior) = REQUIRED]; // Required. The Cloud Dataproc region in which to handle the request. - string region = 5; + string region = 5 [(google.api.field_behavior) = REQUIRED]; // Required. The cluster name. - string cluster_name = 2; + string cluster_name = 2 [(google.api.field_behavior) = REQUIRED]; // Required. The changes to the cluster. - Cluster cluster = 3; + Cluster cluster = 3 [(google.api.field_behavior) = REQUIRED]; // Optional. Timeout for graceful YARN decomissioning. Graceful // decommissioning allows removing nodes from the cluster without @@ -535,7 +672,7 @@ message UpdateClusterRequest { // the maximum allowed timeout is 1 day. // // Only supported on Dataproc image versions 1.2 and higher. - google.protobuf.Duration graceful_decommission_timeout = 6; + google.protobuf.Duration graceful_decommission_timeout = 6 [(google.api.field_behavior) = OPTIONAL]; // Required. Specifies the path, relative to `Cluster`, of // the field to update. For example, to change the number of workers @@ -582,9 +719,13 @@ message UpdateClusterRequest { // config.secondary_worker_config.num_instances // Resize secondary worker group // + // + // config.autoscaling_config.policy_uriUse, stop using, or + // change autoscaling policies + // // // - google.protobuf.FieldMask update_mask = 4; + google.protobuf.FieldMask update_mask = 4 [(google.api.field_behavior) = REQUIRED]; // Optional. A unique id used to identify the request. If the server // receives two [UpdateClusterRequest][google.cloud.dataproc.v1.UpdateClusterRequest] requests with the same @@ -597,24 +738,24 @@ message UpdateClusterRequest { // // The id must contain only letters (a-z, A-Z), numbers (0-9), // underscores (_), and hyphens (-). The maximum length is 40 characters. - string request_id = 7; + string request_id = 7 [(google.api.field_behavior) = OPTIONAL]; } // A request to delete a cluster. message DeleteClusterRequest { // Required. The ID of the Google Cloud Platform project that the cluster // belongs to. - string project_id = 1; + string project_id = 1 [(google.api.field_behavior) = REQUIRED]; // Required. The Cloud Dataproc region in which to handle the request. - string region = 3; + string region = 3 [(google.api.field_behavior) = REQUIRED]; // Required. The cluster name. - string cluster_name = 2; + string cluster_name = 2 [(google.api.field_behavior) = REQUIRED]; // Optional. Specifying the `cluster_uuid` means the RPC should fail // (with error NOT_FOUND) if cluster with specified UUID does not exist. - string cluster_uuid = 4; + string cluster_uuid = 4 [(google.api.field_behavior) = OPTIONAL]; // Optional. A unique id used to identify the request. If the server // receives two [DeleteClusterRequest][google.cloud.dataproc.v1.DeleteClusterRequest] requests with the same @@ -627,30 +768,30 @@ message DeleteClusterRequest { // // The id must contain only letters (a-z, A-Z), numbers (0-9), // underscores (_), and hyphens (-). The maximum length is 40 characters. - string request_id = 5; + string request_id = 5 [(google.api.field_behavior) = OPTIONAL]; } // Request to get the resource representation for a cluster in a project. message GetClusterRequest { // Required. The ID of the Google Cloud Platform project that the cluster // belongs to. - string project_id = 1; + string project_id = 1 [(google.api.field_behavior) = REQUIRED]; // Required. The Cloud Dataproc region in which to handle the request. - string region = 3; + string region = 3 [(google.api.field_behavior) = REQUIRED]; // Required. The cluster name. - string cluster_name = 2; + string cluster_name = 2 [(google.api.field_behavior) = REQUIRED]; } // A request to list the clusters in a project. message ListClustersRequest { // Required. The ID of the Google Cloud Platform project that the cluster // belongs to. - string project_id = 1; + string project_id = 1 [(google.api.field_behavior) = REQUIRED]; // Required. The Cloud Dataproc region in which to handle the request. - string region = 4; + string region = 4 [(google.api.field_behavior) = REQUIRED]; // Optional. A filter constraining the clusters to list. Filters are // case-sensitive and have the following syntax: @@ -671,37 +812,37 @@ message ListClustersRequest { // // status.state = ACTIVE AND clusterName = mycluster // AND labels.env = staging AND labels.starred = * - string filter = 5; + string filter = 5 [(google.api.field_behavior) = OPTIONAL]; // Optional. The standard List page size. - int32 page_size = 2; + int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL]; // Optional. The standard List page token. - string page_token = 3; + string page_token = 3 [(google.api.field_behavior) = OPTIONAL]; } // The list of all clusters in a project. message ListClustersResponse { // Output only. The clusters in the project. - repeated Cluster clusters = 1; + repeated Cluster clusters = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. This token is included in the response if there are more // results to fetch. To fetch additional results, provide this value as the // `page_token` in a subsequent `ListClustersRequest`. - string next_page_token = 2; + string next_page_token = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; } // A request to collect cluster diagnostic information. message DiagnoseClusterRequest { // Required. The ID of the Google Cloud Platform project that the cluster // belongs to. - string project_id = 1; + string project_id = 1 [(google.api.field_behavior) = REQUIRED]; // Required. The Cloud Dataproc region in which to handle the request. - string region = 3; + string region = 3 [(google.api.field_behavior) = REQUIRED]; // Required. The cluster name. - string cluster_name = 2; + string cluster_name = 2 [(google.api.field_behavior) = REQUIRED]; } // The location of diagnostic output. @@ -709,5 +850,5 @@ message DiagnoseClusterResults { // Output only. The Cloud Storage URI of the diagnostic output. // The output report is a plain text file with a summary of collected // diagnostics. - string output_uri = 1; + string output_uri = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; } diff --git a/google/cloud/dataproc/v1/dataproc_v1.yaml b/google/cloud/dataproc/v1/dataproc_v1.yaml new file mode 100644 index 0000000000..c1646e8c05 --- /dev/null +++ b/google/cloud/dataproc/v1/dataproc_v1.yaml @@ -0,0 +1,127 @@ +type: google.api.Service +config_version: 2 +name: dataproc.googleapis.com +title: Cloud Dataproc API + +apis: +- name: google.cloud.dataproc.v1.AutoscalingPolicyService +- name: google.cloud.dataproc.v1.ClusterController +- name: google.cloud.dataproc.v1.JobController +- name: google.cloud.dataproc.v1.WorkflowTemplateService + +types: +- name: google.cloud.dataproc.v1.ClusterOperationMetadata +- name: google.cloud.dataproc.v1.DiagnoseClusterResults +- name: google.cloud.dataproc.v1.WorkflowMetadata + +documentation: + summary: 'Manages Hadoop-based clusters and jobs on Google Cloud Platform.' + rules: + - selector: google.iam.v1.IAMPolicy.GetIamPolicy + description: |- + Gets the access control policy for a resource. Returns an empty policy + if the resource exists and does not have a policy set. + + - selector: google.iam.v1.IAMPolicy.SetIamPolicy + description: |- + Sets the access control policy on the specified resource. Replaces + any existing policy. + + Can return Public Errors: NOT_FOUND, INVALID_ARGUMENT and + PERMISSION_DENIED + + - selector: google.iam.v1.IAMPolicy.TestIamPermissions + description: |- + Returns permissions that a caller has on the specified resource. If the + resource does not exist, this will return an empty set of + permissions, not a NOT_FOUND error. + + Note: This operation is designed to be used for building + permission-aware UIs and command-line tools, not for authorization + checking. This operation may "fail open" without warning. + +http: + rules: + - selector: google.iam.v1.IAMPolicy.GetIamPolicy + post: '/v1/{resource=projects/*/regions/*/clusters/*}:getIamPolicy' + body: '*' + additional_bindings: + - post: '/v1/{resource=projects/*/regions/*/jobs/*}:getIamPolicy' + body: '*' + - post: '/v1/{resource=projects/*/regions/*/operations/*}:getIamPolicy' + body: '*' + - post: '/v1/{resource=projects/*/regions/*/workflowTemplates/*}:getIamPolicy' + body: '*' + - post: '/v1/{resource=projects/*/locations/*/workflowTemplates/*}:getIamPolicy' + body: '*' + - post: '/v1/{resource=projects/*/regions/*/autoscalingPolicies/*}:getIamPolicy' + body: '*' + - post: '/v1/{resource=projects/*/locations/*/autoscalingPolicies/*}:getIamPolicy' + body: '*' + - selector: google.iam.v1.IAMPolicy.SetIamPolicy + post: '/v1/{resource=projects/*/regions/*/clusters/*}:setIamPolicy' + body: '*' + additional_bindings: + - post: '/v1/{resource=projects/*/regions/*/jobs/*}:setIamPolicy' + body: '*' + - post: '/v1/{resource=projects/*/regions/*/operations/*}:setIamPolicy' + body: '*' + - post: '/v1/{resource=projects/*/regions/*/workflowTemplates/*}:setIamPolicy' + body: '*' + - post: '/v1/{resource=projects/*/locations/*/workflowTemplates/*}:setIamPolicy' + body: '*' + - post: '/v1/{resource=projects/*/regions/*/autoscalingPolicies/*}:setIamPolicy' + body: '*' + - post: '/v1/{resource=projects/*/locations/*/autoscalingPolicies/*}:setIamPolicy' + body: '*' + - selector: google.iam.v1.IAMPolicy.TestIamPermissions + post: '/v1/{resource=projects/*/regions/*/clusters/*}:testIamPermissions' + body: '*' + additional_bindings: + - post: '/v1/{resource=projects/*/regions/*/jobs/*}:testIamPermissions' + body: '*' + - post: '/v1/{resource=projects/*/regions/*/operations/*}:testIamPermissions' + body: '*' + - post: '/v1/{resource=projects/*/regions/*/workflowTemplates/*}:testIamPermissions' + body: '*' + - post: '/v1/{resource=projects/*/locations/*/workflowTemplates/*}:testIamPermissions' + body: '*' + - post: '/v1/{resource=projects/*/regions/*/autoscalingPolicies/*}:testIamPermissions' + body: '*' + - post: '/v1/{resource=projects/*/locations/*/autoscalingPolicies/*}:testIamPermissions' + body: '*' + - selector: google.longrunning.Operations.CancelOperation + post: '/v1/{name=projects/*/regions/*/operations/*}:cancel' + - selector: google.longrunning.Operations.DeleteOperation + delete: '/v1/{name=projects/*/regions/*/operations/*}' + - selector: google.longrunning.Operations.GetOperation + get: '/v1/{name=projects/*/regions/*/operations/*}' + - selector: google.longrunning.Operations.ListOperations + get: '/v1/{name=projects/*/regions/*/operations}' + +authentication: + rules: + - selector: 'google.cloud.dataproc.v1.AutoscalingPolicyService.*' + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: 'google.cloud.dataproc.v1.ClusterController.*' + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: 'google.cloud.dataproc.v1.JobController.*' + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: 'google.cloud.dataproc.v1.WorkflowTemplateService.*' + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: 'google.iam.v1.IAMPolicy.*' + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: 'google.longrunning.Operations.*' + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform diff --git a/google/cloud/dataproc/v1/jobs.proto b/google/cloud/dataproc/v1/jobs.proto index 7ead7bb95c..eeba155dee 100644 --- a/google/cloud/dataproc/v1/jobs.proto +++ b/google/cloud/dataproc/v1/jobs.proto @@ -18,6 +18,8 @@ syntax = "proto3"; package google.cloud.dataproc.v1; import "google/api/annotations.proto"; +import "google/api/client.proto"; +import "google/api/field_behavior.proto"; import "google/protobuf/empty.proto"; import "google/protobuf/field_mask.proto"; import "google/protobuf/timestamp.proto"; @@ -29,12 +31,16 @@ option java_package = "com.google.cloud.dataproc.v1"; // The JobController provides methods to manage jobs. service JobController { + option (google.api.default_host) = "dataproc.googleapis.com"; + option (google.api.oauth_scopes) = "https://www.googleapis.com/auth/cloud-platform"; + // Submits a job to a cluster. rpc SubmitJob(SubmitJobRequest) returns (Job) { option (google.api.http) = { post: "/v1/projects/{project_id}/regions/{region}/jobs:submit" body: "*" }; + option (google.api.method_signature) = "project_id,region,job"; } // Gets the resource representation for a job in a project. @@ -42,6 +48,7 @@ service JobController { option (google.api.http) = { get: "/v1/projects/{project_id}/regions/{region}/jobs/{job_id}" }; + option (google.api.method_signature) = "project_id,region,job_id"; } // Lists regions/{region}/jobs in a project. @@ -49,6 +56,8 @@ service JobController { option (google.api.http) = { get: "/v1/projects/{project_id}/regions/{region}/jobs" }; + option (google.api.method_signature) = "project_id,region"; + option (google.api.method_signature) = "project_id,region,filter"; } // Updates a job in a project. @@ -69,6 +78,7 @@ service JobController { post: "/v1/projects/{project_id}/regions/{region}/jobs/{job_id}:cancel" body: "*" }; + option (google.api.method_signature) = "project_id,region,job_id"; } // Deletes the job from the project. If the job is active, the delete fails, @@ -77,6 +87,7 @@ service JobController { option (google.api.http) = { delete: "/v1/projects/{project_id}/regions/{region}/jobs/{job_id}" }; + option (google.api.method_signature) = "project_id,region,job_id"; } } @@ -148,30 +159,30 @@ message HadoopJob { // include arguments, such as `-libjars` or `-Dfoo=bar`, that can be set as // job properties, since a collision may occur that causes an incorrect job // submission. - repeated string args = 3; + repeated string args = 3 [(google.api.field_behavior) = OPTIONAL]; // Optional. Jar file URIs to add to the CLASSPATHs of the // Hadoop driver and tasks. - repeated string jar_file_uris = 4; + repeated string jar_file_uris = 4 [(google.api.field_behavior) = OPTIONAL]; // Optional. HCFS (Hadoop Compatible Filesystem) URIs of files to be copied // to the working directory of Hadoop drivers and distributed tasks. Useful // for naively parallel tasks. - repeated string file_uris = 5; + repeated string file_uris = 5 [(google.api.field_behavior) = OPTIONAL]; // Optional. HCFS URIs of archives to be extracted in the working directory of // Hadoop drivers and tasks. Supported file types: // .jar, .tar, .tar.gz, .tgz, or .zip. - repeated string archive_uris = 6; + repeated string archive_uris = 6 [(google.api.field_behavior) = OPTIONAL]; // Optional. A mapping of property names to values, used to configure Hadoop. // Properties that conflict with values set by the Cloud Dataproc API may be // overwritten. Can include properties set in /etc/hadoop/conf/*-site and // classes in user code. - map properties = 7; + map properties = 7 [(google.api.field_behavior) = OPTIONAL]; // Optional. The runtime log config for job execution. - LoggingConfig logging_config = 8; + LoggingConfig logging_config = 8 [(google.api.field_behavior) = OPTIONAL]; } // A Cloud Dataproc job for running [Apache Spark](http://spark.apache.org/) @@ -194,29 +205,29 @@ message SparkJob { // Optional. The arguments to pass to the driver. Do not include arguments, // such as `--conf`, that can be set as job properties, since a collision may // occur that causes an incorrect job submission. - repeated string args = 3; + repeated string args = 3 [(google.api.field_behavior) = OPTIONAL]; // Optional. HCFS URIs of jar files to add to the CLASSPATHs of the // Spark driver and tasks. - repeated string jar_file_uris = 4; + repeated string jar_file_uris = 4 [(google.api.field_behavior) = OPTIONAL]; // Optional. HCFS URIs of files to be copied to the working directory of // Spark drivers and distributed tasks. Useful for naively parallel tasks. - repeated string file_uris = 5; + repeated string file_uris = 5 [(google.api.field_behavior) = OPTIONAL]; // Optional. HCFS URIs of archives to be extracted in the working directory // of Spark drivers and tasks. Supported file types: // .jar, .tar, .tar.gz, .tgz, and .zip. - repeated string archive_uris = 6; + repeated string archive_uris = 6 [(google.api.field_behavior) = OPTIONAL]; // Optional. A mapping of property names to values, used to configure Spark. // Properties that conflict with values set by the Cloud Dataproc API may be // overwritten. Can include properties set in // /etc/spark/conf/spark-defaults.conf and classes in user code. - map properties = 7; + map properties = 7 [(google.api.field_behavior) = OPTIONAL]; // Optional. The runtime log config for job execution. - LoggingConfig logging_config = 8; + LoggingConfig logging_config = 8 [(google.api.field_behavior) = OPTIONAL]; } // A Cloud Dataproc job for running @@ -226,37 +237,37 @@ message SparkJob { message PySparkJob { // Required. The HCFS URI of the main Python file to use as the driver. Must // be a .py file. - string main_python_file_uri = 1; + string main_python_file_uri = 1 [(google.api.field_behavior) = REQUIRED]; // Optional. The arguments to pass to the driver. Do not include arguments, // such as `--conf`, that can be set as job properties, since a collision may // occur that causes an incorrect job submission. - repeated string args = 2; + repeated string args = 2 [(google.api.field_behavior) = OPTIONAL]; // Optional. HCFS file URIs of Python files to pass to the PySpark // framework. Supported file types: .py, .egg, and .zip. - repeated string python_file_uris = 3; + repeated string python_file_uris = 3 [(google.api.field_behavior) = OPTIONAL]; // Optional. HCFS URIs of jar files to add to the CLASSPATHs of the // Python driver and tasks. - repeated string jar_file_uris = 4; + repeated string jar_file_uris = 4 [(google.api.field_behavior) = OPTIONAL]; // Optional. HCFS URIs of files to be copied to the working directory of // Python drivers and distributed tasks. Useful for naively parallel tasks. - repeated string file_uris = 5; + repeated string file_uris = 5 [(google.api.field_behavior) = OPTIONAL]; // Optional. HCFS URIs of archives to be extracted in the working directory of // .jar, .tar, .tar.gz, .tgz, and .zip. - repeated string archive_uris = 6; + repeated string archive_uris = 6 [(google.api.field_behavior) = OPTIONAL]; // Optional. A mapping of property names to values, used to configure PySpark. // Properties that conflict with values set by the Cloud Dataproc API may be // overwritten. Can include properties set in // /etc/spark/conf/spark-defaults.conf and classes in user code. - map properties = 7; + map properties = 7 [(google.api.field_behavior) = OPTIONAL]; // Optional. The runtime log config for job execution. - LoggingConfig logging_config = 8; + LoggingConfig logging_config = 8 [(google.api.field_behavior) = OPTIONAL]; } // A list of queries to run on a cluster. @@ -275,7 +286,7 @@ message QueryList { // ] // } // } - repeated string queries = 1; + repeated string queries = 1 [(google.api.field_behavior) = REQUIRED]; } // A Cloud Dataproc job for running [Apache Hive](https://hive.apache.org/) @@ -294,22 +305,22 @@ message HiveJob { // Optional. Whether to continue executing queries if a query fails. // The default value is `false`. Setting to `true` can be useful when // executing independent parallel queries. - bool continue_on_failure = 3; + bool continue_on_failure = 3 [(google.api.field_behavior) = OPTIONAL]; // Optional. Mapping of query variable names to values (equivalent to the // Hive command: `SET name="value";`). - map script_variables = 4; + map script_variables = 4 [(google.api.field_behavior) = OPTIONAL]; // Optional. A mapping of property names and values, used to configure Hive. // Properties that conflict with values set by the Cloud Dataproc API may be // overwritten. Can include properties set in /etc/hadoop/conf/*-site.xml, // /etc/hive/conf/hive-site.xml, and classes in user code. - map properties = 5; + map properties = 5 [(google.api.field_behavior) = OPTIONAL]; // Optional. HCFS URIs of jar files to add to the CLASSPATH of the // Hive server and Hadoop MapReduce (MR) tasks. Can contain Hive SerDes // and UDFs. - repeated string jar_file_uris = 6; + repeated string jar_file_uris = 6 [(google.api.field_behavior) = OPTIONAL]; } // A Cloud Dataproc job for running [Apache Spark @@ -327,18 +338,18 @@ message SparkSqlJob { // Optional. Mapping of query variable names to values (equivalent to the // Spark SQL command: SET `name="value";`). - map script_variables = 3; + map script_variables = 3 [(google.api.field_behavior) = OPTIONAL]; // Optional. A mapping of property names to values, used to configure // Spark SQL's SparkConf. Properties that conflict with values set by the // Cloud Dataproc API may be overwritten. - map properties = 4; + map properties = 4 [(google.api.field_behavior) = OPTIONAL]; // Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH. - repeated string jar_file_uris = 56; + repeated string jar_file_uris = 56 [(google.api.field_behavior) = OPTIONAL]; // Optional. The runtime log config for job execution. - LoggingConfig logging_config = 6; + LoggingConfig logging_config = 6 [(google.api.field_behavior) = OPTIONAL]; } // A Cloud Dataproc job for running [Apache Pig](https://pig.apache.org/) @@ -357,34 +368,34 @@ message PigJob { // Optional. Whether to continue executing queries if a query fails. // The default value is `false`. Setting to `true` can be useful when // executing independent parallel queries. - bool continue_on_failure = 3; + bool continue_on_failure = 3 [(google.api.field_behavior) = OPTIONAL]; // Optional. Mapping of query variable names to values (equivalent to the Pig // command: `name=[value]`). - map script_variables = 4; + map script_variables = 4 [(google.api.field_behavior) = OPTIONAL]; // Optional. A mapping of property names to values, used to configure Pig. // Properties that conflict with values set by the Cloud Dataproc API may be // overwritten. Can include properties set in /etc/hadoop/conf/*-site.xml, // /etc/pig/conf/pig.properties, and classes in user code. - map properties = 5; + map properties = 5 [(google.api.field_behavior) = OPTIONAL]; // Optional. HCFS URIs of jar files to add to the CLASSPATH of // the Pig Client and Hadoop MapReduce (MR) tasks. Can contain Pig UDFs. - repeated string jar_file_uris = 6; + repeated string jar_file_uris = 6 [(google.api.field_behavior) = OPTIONAL]; // Optional. The runtime log config for job execution. - LoggingConfig logging_config = 7; + LoggingConfig logging_config = 7 [(google.api.field_behavior) = OPTIONAL]; } // Cloud Dataproc job config. message JobPlacement { // Required. The name of the cluster where the job will be submitted. - string cluster_name = 1; + string cluster_name = 1 [(google.api.field_behavior) = REQUIRED]; // Output only. A cluster UUID generated by the Cloud Dataproc service when // the job is submitted. - string cluster_uuid = 2; + string cluster_uuid = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Cloud Dataproc job status. @@ -453,25 +464,28 @@ message JobStatus { } // Output only. A state message specifying the overall job state. - State state = 1; + State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; - // Output only. Optional job state details, such as an error + // Optional. Output only. Job state details, such as an error // description if the state is ERROR. - string details = 2; + string details = 2 [ + (google.api.field_behavior) = OUTPUT_ONLY, + (google.api.field_behavior) = OPTIONAL + ]; // Output only. The time when this state was entered. - google.protobuf.Timestamp state_start_time = 6; + google.protobuf.Timestamp state_start_time = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Additional state information, which includes // status reported by the agent. - Substate substate = 7; + Substate substate = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Encapsulates the full scoping used to reference a job. message JobReference { // Required. The ID of the Google Cloud Platform project that the job // belongs to. - string project_id = 1; + string project_id = 1 [(google.api.field_behavior) = REQUIRED]; // Optional. The job ID, which must be unique within the project. // @@ -479,7 +493,7 @@ message JobReference { // underscores (_), or hyphens (-). The maximum length is 100 characters. // // If not specified by the caller, the job ID will be provided by the server. - string job_id = 2; + string job_id = 2 [(google.api.field_behavior) = OPTIONAL]; } // A YARN application created by a job. Application information is a subset of @@ -520,19 +534,19 @@ message YarnApplication { } // Required. The application name. - string name = 1; + string name = 1 [(google.api.field_behavior) = REQUIRED]; // Required. The application state. - State state = 2; + State state = 2 [(google.api.field_behavior) = REQUIRED]; // Required. The numerical progress of the application, from 1 to 100. - float progress = 3; + float progress = 3 [(google.api.field_behavior) = REQUIRED]; // Optional. The HTTP URL of the ApplicationMaster, HistoryServer, or // TimelineServer that provides application-specific information. The URL uses // the internal hostname, and requires a proxy server for resolution and, // possibly, access. - string tracking_url = 4; + string tracking_url = 4 [(google.api.field_behavior) = OPTIONAL]; } // A Cloud Dataproc job resource. @@ -541,11 +555,11 @@ message Job { // obtain the equivalent REST path of the job resource. If this property // is not specified when a job is created, the server generates a // job_id. - JobReference reference = 1; + JobReference reference = 1 [(google.api.field_behavior) = OPTIONAL]; // Required. Job information, including how, when, and where to // run the job. - JobPlacement placement = 2; + JobPlacement placement = 2 [(google.api.field_behavior) = REQUIRED]; // Required. The application/framework-specific portion of the job. oneof type_job { @@ -571,25 +585,25 @@ message Job { // Output only. The job status. Additional application-specific // status information may be contained in the type_job // and yarn_applications fields. - JobStatus status = 8; + JobStatus status = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The previous job status. - repeated JobStatus status_history = 13; + repeated JobStatus status_history = 13 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The collection of YARN applications spun up by this job. // // **Beta** Feature: This report is available for testing purposes only. It // may be changed before final release. - repeated YarnApplication yarn_applications = 9; + repeated YarnApplication yarn_applications = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. A URI pointing to the location of the stdout of the job's // driver program. - string driver_output_resource_uri = 17; + string driver_output_resource_uri = 17 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. If present, the location of miscellaneous control files // which may be used as part of job setup and handling. If not present, // control files may be placed in the same location as `driver_output_uri`. - string driver_control_files_uri = 15; + string driver_control_files_uri = 15 [(google.api.field_behavior) = OUTPUT_ONLY]; // Optional. The labels to associate with this job. // Label **keys** must contain 1 to 63 characters, and must conform to @@ -598,15 +612,15 @@ message Job { // characters, and must conform to [RFC // 1035](https://www.ietf.org/rfc/rfc1035.txt). No more than 32 labels can be // associated with a job. - map labels = 18; + map labels = 18 [(google.api.field_behavior) = OPTIONAL]; // Optional. Job scheduling configuration. - JobScheduling scheduling = 20; + JobScheduling scheduling = 20 [(google.api.field_behavior) = OPTIONAL]; // Output only. A UUID that uniquely identifies a job within the project // over time. This is in contrast to a user-settable reference.job_id that // may be reused over time. - string job_uuid = 22; + string job_uuid = 22 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Job scheduling options. @@ -619,20 +633,20 @@ message JobScheduling { // 4 times within 10 minute window. // // Maximum value is 10. - int32 max_failures_per_hour = 1; + int32 max_failures_per_hour = 1 [(google.api.field_behavior) = OPTIONAL]; } // A request to submit a job. message SubmitJobRequest { // Required. The ID of the Google Cloud Platform project that the job // belongs to. - string project_id = 1; + string project_id = 1 [(google.api.field_behavior) = REQUIRED]; // Required. The Cloud Dataproc region in which to handle the request. - string region = 3; + string region = 3 [(google.api.field_behavior) = REQUIRED]; // Required. The job resource. - Job job = 2; + Job job = 2 [(google.api.field_behavior) = REQUIRED]; // Optional. A unique id used to identify the request. If the server // receives two [SubmitJobRequest][google.cloud.dataproc.v1.SubmitJobRequest] requests with the same @@ -645,20 +659,20 @@ message SubmitJobRequest { // // The id must contain only letters (a-z, A-Z), numbers (0-9), // underscores (_), and hyphens (-). The maximum length is 40 characters. - string request_id = 4; + string request_id = 4 [(google.api.field_behavior) = OPTIONAL]; } // A request to get the resource representation for a job in a project. message GetJobRequest { // Required. The ID of the Google Cloud Platform project that the job // belongs to. - string project_id = 1; + string project_id = 1 [(google.api.field_behavior) = REQUIRED]; // Required. The Cloud Dataproc region in which to handle the request. - string region = 3; + string region = 3 [(google.api.field_behavior) = REQUIRED]; // Required. The job ID. - string job_id = 2; + string job_id = 2 [(google.api.field_behavior) = REQUIRED]; } // A request to list jobs in a project. @@ -678,27 +692,27 @@ message ListJobsRequest { // Required. The ID of the Google Cloud Platform project that the job // belongs to. - string project_id = 1; + string project_id = 1 [(google.api.field_behavior) = REQUIRED]; // Required. The Cloud Dataproc region in which to handle the request. - string region = 6; + string region = 6 [(google.api.field_behavior) = REQUIRED]; // Optional. The number of results to return in each response. - int32 page_size = 2; + int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL]; // Optional. The page token, returned by a previous call, to request the // next page of results. - string page_token = 3; + string page_token = 3 [(google.api.field_behavior) = OPTIONAL]; // Optional. If set, the returned jobs list includes only jobs that were // submitted to the named cluster. - string cluster_name = 4; + string cluster_name = 4 [(google.api.field_behavior) = OPTIONAL]; // Optional. Specifies enumerated categories of jobs to list. // (default = match ALL jobs). // // If `filter` is provided, `jobStateMatcher` will be ignored. - JobStateMatcher job_state_matcher = 5; + JobStateMatcher job_state_matcher = 5 [(google.api.field_behavior) = OPTIONAL]; // Optional. A filter constraining the jobs to list. Filters are // case-sensitive and have the following syntax: @@ -714,23 +728,23 @@ message ListJobsRequest { // Example filter: // // status.state = ACTIVE AND labels.env = staging AND labels.starred = * - string filter = 7; + string filter = 7 [(google.api.field_behavior) = OPTIONAL]; } // A request to update a job. message UpdateJobRequest { // Required. The ID of the Google Cloud Platform project that the job // belongs to. - string project_id = 1; + string project_id = 1 [(google.api.field_behavior) = REQUIRED]; // Required. The Cloud Dataproc region in which to handle the request. - string region = 2; + string region = 2 [(google.api.field_behavior) = REQUIRED]; // Required. The job ID. - string job_id = 3; + string job_id = 3 [(google.api.field_behavior) = REQUIRED]; // Required. The changes to the job. - Job job = 4; + Job job = 4 [(google.api.field_behavior) = REQUIRED]; // Required. Specifies the path, relative to Job, of // the field to update. For example, to update the labels of a Job the @@ -738,42 +752,42 @@ message UpdateJobRequest { // labels, and the `PATCH` request body would specify the new // value. Note: Currently, labels is the only // field that can be updated. - google.protobuf.FieldMask update_mask = 5; + google.protobuf.FieldMask update_mask = 5 [(google.api.field_behavior) = REQUIRED]; } // A list of jobs in a project. message ListJobsResponse { // Output only. Jobs list. - repeated Job jobs = 1; + repeated Job jobs = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Optional. This token is included in the response if there are more results // to fetch. To fetch additional results, provide this value as the // `page_token` in a subsequent ListJobsRequest. - string next_page_token = 2; + string next_page_token = 2 [(google.api.field_behavior) = OPTIONAL]; } // A request to cancel a job. message CancelJobRequest { // Required. The ID of the Google Cloud Platform project that the job // belongs to. - string project_id = 1; + string project_id = 1 [(google.api.field_behavior) = REQUIRED]; // Required. The Cloud Dataproc region in which to handle the request. - string region = 3; + string region = 3 [(google.api.field_behavior) = REQUIRED]; // Required. The job ID. - string job_id = 2; + string job_id = 2 [(google.api.field_behavior) = REQUIRED]; } // A request to delete a job. message DeleteJobRequest { // Required. The ID of the Google Cloud Platform project that the job // belongs to. - string project_id = 1; + string project_id = 1 [(google.api.field_behavior) = REQUIRED]; // Required. The Cloud Dataproc region in which to handle the request. - string region = 3; + string region = 3 [(google.api.field_behavior) = REQUIRED]; // Required. The job ID. - string job_id = 2; + string job_id = 2 [(google.api.field_behavior) = REQUIRED]; } diff --git a/google/cloud/dataproc/v1/operations.proto b/google/cloud/dataproc/v1/operations.proto index c820cd8e65..4af2a5f807 100644 --- a/google/cloud/dataproc/v1/operations.proto +++ b/google/cloud/dataproc/v1/operations.proto @@ -17,8 +17,9 @@ syntax = "proto3"; package google.cloud.dataproc.v1; -import "google/api/annotations.proto"; +import "google/api/field_behavior.proto"; import "google/protobuf/timestamp.proto"; +import "google/api/annotations.proto"; option go_package = "google.golang.org/genproto/googleapis/cloud/dataproc/v1;dataproc"; option java_multiple_files = true; @@ -43,41 +44,41 @@ message ClusterOperationStatus { } // Output only. A message containing the operation state. - State state = 1; + State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. A message containing the detailed operation state. - string inner_state = 2; + string inner_state = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. A message containing any operation metadata details. - string details = 3; + string details = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The time this state was entered. - google.protobuf.Timestamp state_start_time = 4; + google.protobuf.Timestamp state_start_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Metadata describing the operation. message ClusterOperationMetadata { // Output only. Name of the cluster for the operation. - string cluster_name = 7; + string cluster_name = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Cluster UUID for the operation. - string cluster_uuid = 8; + string cluster_uuid = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Current operation status. - ClusterOperationStatus status = 9; + ClusterOperationStatus status = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The previous operation status. - repeated ClusterOperationStatus status_history = 10; + repeated ClusterOperationStatus status_history = 10 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The operation type. - string operation_type = 11; + string operation_type = 11 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Short description of operation. - string description = 12; + string description = 12 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Labels associated with the operation - map labels = 13; + map labels = 13 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Errors encountered during operation execution. - repeated string warnings = 14; + repeated string warnings = 14 [(google.api.field_behavior) = OUTPUT_ONLY]; } diff --git a/google/cloud/dataproc/v1/workflow_templates.proto b/google/cloud/dataproc/v1/workflow_templates.proto index 61295a5500..8976c42e29 100644 --- a/google/cloud/dataproc/v1/workflow_templates.proto +++ b/google/cloud/dataproc/v1/workflow_templates.proto @@ -18,6 +18,9 @@ syntax = "proto3"; package google.cloud.dataproc.v1; import "google/api/annotations.proto"; +import "google/api/client.proto"; +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; import "google/cloud/dataproc/v1/clusters.proto"; import "google/cloud/dataproc/v1/jobs.proto"; import "google/longrunning/operations.proto"; @@ -32,6 +35,9 @@ option java_package = "com.google.cloud.dataproc.v1"; // The API interface for managing Workflow Templates in the // Cloud Dataproc API. service WorkflowTemplateService { + option (google.api.default_host) = "dataproc.googleapis.com"; + option (google.api.oauth_scopes) = "https://www.googleapis.com/auth/cloud-platform"; + // Creates new workflow template. rpc CreateWorkflowTemplate(CreateWorkflowTemplateRequest) returns (WorkflowTemplate) { option (google.api.http) = { @@ -42,6 +48,7 @@ service WorkflowTemplateService { body: "template" } }; + option (google.api.method_signature) = "parent,template"; } // Retrieves the latest workflow template. @@ -55,6 +62,7 @@ service WorkflowTemplateService { get: "/v1/{name=projects/*/regions/*/workflowTemplates/*}" } }; + option (google.api.method_signature) = "name"; } // Instantiates a template and begins execution. @@ -70,7 +78,9 @@ service WorkflowTemplateService { // clusters to be deleted. // // The [Operation.metadata][google.longrunning.Operation.metadata] will be - // [WorkflowMetadata][google.cloud.dataproc.v1.WorkflowMetadata]. + // [WorkflowMetadata](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#workflowmetadata). + // Also see [Using + // WorkflowMetadata](/dataproc/docs/concepts/workflows/debugging#using_workflowmetadata). // // On successful completion, // [Operation.response][google.longrunning.Operation.response] will be @@ -84,6 +94,12 @@ service WorkflowTemplateService { body: "*" } }; + option (google.api.method_signature) = "name"; + option (google.api.method_signature) = "name,parameters"; + option (google.longrunning.operation_info) = { + response_type: "google.protobuf.Empty" + metadata_type: "WorkflowMetadata" + }; } // Instantiates a template and begins execution. @@ -103,7 +119,9 @@ service WorkflowTemplateService { // clusters to be deleted. // // The [Operation.metadata][google.longrunning.Operation.metadata] will be - // [WorkflowMetadata][google.cloud.dataproc.v1.WorkflowMetadata]. + // [WorkflowMetadata](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#workflowmetadata). + // Also see [Using + // WorkflowMetadata](/dataproc/docs/concepts/workflows/debugging#using_workflowmetadata). // // On successful completion, // [Operation.response][google.longrunning.Operation.response] will be @@ -117,6 +135,11 @@ service WorkflowTemplateService { body: "template" } }; + option (google.api.method_signature) = "parent,template"; + option (google.longrunning.operation_info) = { + response_type: "google.protobuf.Empty" + metadata_type: "WorkflowMetadata" + }; } // Updates (replaces) workflow template. The updated template @@ -130,6 +153,7 @@ service WorkflowTemplateService { body: "template" } }; + option (google.api.method_signature) = "template"; } // Lists workflows that match the specified filter in the request. @@ -140,6 +164,7 @@ service WorkflowTemplateService { get: "/v1/{parent=projects/*/regions/*}/workflowTemplates" } }; + option (google.api.method_signature) = "parent"; } // Deletes a workflow template. It does not cancel in-progress workflows. @@ -150,22 +175,32 @@ service WorkflowTemplateService { delete: "/v1/{name=projects/*/regions/*/workflowTemplates/*}" } }; + option (google.api.method_signature) = "name"; } } // A Cloud Dataproc workflow template resource. message WorkflowTemplate { - // Required. The template id. - // - // The id must contain only letters (a-z, A-Z), numbers (0-9), - // underscores (_), and hyphens (-). Cannot begin or end with underscore - // or hyphen. Must consist of between 3 and 50 characters. - string id = 2; + option (google.api.resource) = { + type: "dataproc.googleapis.com/WorkflowTemplate" + pattern: "projects/{project}/regions/{region}/workflowTemplates/{workflow_template}" + pattern: "projects/{project}/locations/{location}/workflowTemplates/{workflow_template}" + history: ORIGINALLY_SINGLE_PATTERN + }; - // Output only. The "resource name" of the template, as described - // in https://cloud.google.com/apis/design/resource_names of the form - // `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}` - string name = 1; + string id = 2 [(google.api.field_behavior) = REQUIRED]; + + // Output only. The resource name of the workflow template, as described + // in https://cloud.google.com/apis/design/resource_names. + // + // * For `projects.regions.workflowTemplates`, the resource name of the + // template has the following format: + // `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}` + // + // * For `projects.locations.workflowTemplates`, the resource name of the + // template has the following format: + // `projects/{project_id}/locations/{location}/workflowTemplates/{template_id}` + string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Optional. Used to perform a consistent read-modify-write. // @@ -176,13 +211,13 @@ message WorkflowTemplate { // the current template with the `version` field filled in with the // current server version. The user updates other fields in the template, // then returns it as part of the `UpdateWorkflowTemplate` request. - int32 version = 3; + int32 version = 3 [(google.api.field_behavior) = OPTIONAL]; // Output only. The time template was created. - google.protobuf.Timestamp create_time = 4; + google.protobuf.Timestamp create_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The time template was last updated. - google.protobuf.Timestamp update_time = 5; + google.protobuf.Timestamp update_time = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; // Optional. The labels to associate with this template. These labels // will be propagated to all jobs and clusters created by the workflow @@ -196,18 +231,18 @@ message WorkflowTemplate { // [RFC 1035](https://www.ietf.org/rfc/rfc1035.txt). // // No more than 32 labels can be associated with a template. - map labels = 6; + map labels = 6 [(google.api.field_behavior) = OPTIONAL]; // Required. WorkflowTemplate scheduling information. - WorkflowTemplatePlacement placement = 7; + WorkflowTemplatePlacement placement = 7 [(google.api.field_behavior) = REQUIRED]; // Required. The Directed Acyclic Graph of Jobs to submit. - repeated OrderedJob jobs = 8; + repeated OrderedJob jobs = 8 [(google.api.field_behavior) = REQUIRED]; - // Optional. Template parameters whose values are substituted into the + // Optional. emplate parameters whose values are substituted into the // template. Values for parameters must be provided when the template is // instantiated. - repeated TemplateParameter parameters = 9; + repeated TemplateParameter parameters = 9 [(google.api.field_behavior) = OPTIONAL]; } // Specifies workflow execution target. @@ -217,7 +252,7 @@ message WorkflowTemplatePlacement { // Required. Specifies where workflow executes; either on a managed // cluster or an existing cluster chosen by labels. oneof placement { - // Optional. A cluster that is managed by the workflow. + // A cluster that is managed by the workflow. ManagedCluster managed_cluster = 1; // Optional. A selector that chooses target cluster for jobs based @@ -236,10 +271,10 @@ message ManagedCluster { // The name must contain only lower-case letters (a-z), numbers (0-9), // and hyphens (-). Must begin with a letter. Cannot begin or end with // hyphen. Must consist of between 2 and 35 characters. - string cluster_name = 2; + string cluster_name = 2 [(google.api.field_behavior) = REQUIRED]; // Required. The cluster configuration. - ClusterConfig config = 3; + ClusterConfig config = 3 [(google.api.field_behavior) = REQUIRED]; // Optional. The labels to associate with this cluster. // @@ -251,7 +286,7 @@ message ManagedCluster { // the following PCRE regular expression: [\p{Ll}\p{Lo}\p{N}_-]{0,63} // // No more than 32 labels can be associated with a given cluster. - map labels = 4; + map labels = 4 [(google.api.field_behavior) = OPTIONAL]; } // A selector that chooses target cluster for jobs based on metadata. @@ -261,11 +296,11 @@ message ClusterSelector { // // If unspecified, the zone of the first cluster matching the selector // is used. - string zone = 1; + string zone = 1 [(google.api.field_behavior) = OPTIONAL]; // Required. The cluster labels. Cluster must have all labels // to match. - map cluster_labels = 2; + map cluster_labels = 2 [(google.api.field_behavior) = REQUIRED]; } // A job executed by the workflow. @@ -281,7 +316,7 @@ message OrderedJob { // The id must contain only letters (a-z, A-Z), numbers (0-9), // underscores (_), and hyphens (-). Cannot begin or end with underscore // or hyphen. Must consist of between 3 and 50 characters. - string step_id = 1; + string step_id = 1 [(google.api.field_behavior) = REQUIRED]; // Required. The job definition. oneof job_type { @@ -314,14 +349,14 @@ message OrderedJob { // the following regular expression: [\p{Ll}\p{Lo}\p{N}_-]{0,63} // // No more than 32 labels can be associated with a given job. - map labels = 8; + map labels = 8 [(google.api.field_behavior) = OPTIONAL]; // Optional. Job scheduling configuration. - JobScheduling scheduling = 9; + JobScheduling scheduling = 9 [(google.api.field_behavior) = OPTIONAL]; // Optional. The optional list of prerequisite job step_ids. // If not specified, the job will start at the beginning of workflow. - repeated string prerequisite_step_ids = 10; + repeated string prerequisite_step_ids = 10 [(google.api.field_behavior) = OPTIONAL]; } // A configurable parameter that replaces one or more fields in the template. @@ -334,14 +369,14 @@ message OrderedJob { // - Main class (in HadoopJob and SparkJob) // - Zone (in ClusterSelector) message TemplateParameter { - // Required. Parameter name. + // Required. Parameter name. // The parameter name is used as the key, and paired with the // parameter value, which are passed to the template when the template // is instantiated. // The name must contain only capital letters (A-Z), numbers (0-9), and // underscores (_), and must not start with a number. The maximum length is // 40 characters. - string name = 1; + string name = 1 [(google.api.field_behavior) = REQUIRED]; // Required. Paths to all fields that the parameter replaces. // A field is allowed to appear in at most one parameter's list of field @@ -387,14 +422,14 @@ message TemplateParameter { // // - placement.clusterSelector.clusterLabels // - jobs['step-id'].sparkJob.args - repeated string fields = 2; + repeated string fields = 2 [(google.api.field_behavior) = REQUIRED]; // Optional. Brief description of the parameter. // Must not exceed 1024 characters. - string description = 3; + string description = 3 [(google.api.field_behavior) = OPTIONAL]; // Optional. Validation rules to be applied to this parameter's value. - ParameterValidation validation = 4; + ParameterValidation validation = 4 [(google.api.field_behavior) = OPTIONAL]; } // Configuration for parameter validation. @@ -414,13 +449,13 @@ message RegexValidation { // Required. RE2 regular expressions used to validate the parameter's value. // The value must match the regex in its entirety (substring // matches are not sufficient). - repeated string regexes = 1; + repeated string regexes = 1 [(google.api.field_behavior) = REQUIRED]; } // Validation based on a list of allowed values. message ValueValidation { // Required. List of allowed values for the parameter. - repeated string values = 1; + repeated string values = 1 [(google.api.field_behavior) = REQUIRED]; } // A Cloud Dataproc workflow template resource. @@ -440,57 +475,66 @@ message WorkflowMetadata { DONE = 3; } - // Output only. The "resource name" of the template. - string template = 1; + // Output only. The resource name of the workflow template as described + // in https://cloud.google.com/apis/design/resource_names. + // + // * For `projects.regions.workflowTemplates`, the resource name of the + // template has the following format: + // `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}` + // + // * For `projects.locations.workflowTemplates`, the resource name of the + // template has the following format: + // `projects/{project_id}/locations/{location}/workflowTemplates/{template_id}` + string template = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The version of template at the time of // workflow instantiation. - int32 version = 2; + int32 version = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The create cluster operation metadata. - ClusterOperation create_cluster = 3; + ClusterOperation create_cluster = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The workflow graph. - WorkflowGraph graph = 4; + WorkflowGraph graph = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The delete cluster operation metadata. - ClusterOperation delete_cluster = 5; + ClusterOperation delete_cluster = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The workflow state. - State state = 6; + State state = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The name of the target cluster. - string cluster_name = 7; + string cluster_name = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; // Map from parameter names to values that were used for those parameters. map parameters = 8; // Output only. Workflow start time. - google.protobuf.Timestamp start_time = 9; + google.protobuf.Timestamp start_time = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Workflow end time. - google.protobuf.Timestamp end_time = 10; + google.protobuf.Timestamp end_time = 10 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The UUID of target cluster. - string cluster_uuid = 11; + string cluster_uuid = 11 [(google.api.field_behavior) = OUTPUT_ONLY]; } // The cluster operation triggered by a workflow. message ClusterOperation { // Output only. The id of the cluster operation. - string operation_id = 1; + string operation_id = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Error, if operation failed. - string error = 2; + string error = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Indicates the operation is done. - bool done = 3; + bool done = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; } // The workflow graph. message WorkflowGraph { // Output only. The workflow nodes. - repeated WorkflowNode nodes = 1; + repeated WorkflowNode nodes = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; } // The workflow node. @@ -518,52 +562,88 @@ message WorkflowNode { } // Output only. The name of the node. - string step_id = 1; + string step_id = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Node's prerequisite nodes. - repeated string prerequisite_step_ids = 2; + repeated string prerequisite_step_ids = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The job id; populated after the node enters RUNNING state. - string job_id = 3; + string job_id = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The node state. - NodeState state = 5; + NodeState state = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The error detail. - string error = 6; + string error = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; } // A request to create a workflow template. message CreateWorkflowTemplateRequest { - // Required. The "resource name" of the region, as described - // in https://cloud.google.com/apis/design/resource_names of the form - // `projects/{project_id}/regions/{region}` - string parent = 1; + // Required. The resource name of the region or location, as described + // in https://cloud.google.com/apis/design/resource_names. + // + // * For `projects.regions.workflowTemplates,create`, the resource name of the + // region has the following format: + // `projects/{project_id}/regions/{region}` + // + // * For `projects.locations.workflowTemplates.create`, the resource name of + // the location has the following format: + // `projects/{project_id}/locations/{location}` + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + child_type: "dataproc.googleapis.com/WorkflowTemplate" + } + ]; // Required. The Dataproc workflow template to create. - WorkflowTemplate template = 2; + WorkflowTemplate template = 2 [(google.api.field_behavior) = REQUIRED]; } // A request to fetch a workflow template. message GetWorkflowTemplateRequest { - // Required. The "resource name" of the workflow template, as described - // in https://cloud.google.com/apis/design/resource_names of the form - // `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}` - string name = 1; + // Required. The resource name of the workflow template, as described + // in https://cloud.google.com/apis/design/resource_names. + // + // * For `projects.regions.workflowTemplates.get`, the resource name of the + // template has the following format: + // `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}` + // + // * For `projects.locations.workflowTemplates.get`, the resource name of the + // template has the following format: + // `projects/{project_id}/locations/{location}/workflowTemplates/{template_id}` + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "dataproc.googleapis.com/WorkflowTemplate" + } + ]; // Optional. The version of workflow template to retrieve. Only previously - // instatiated versions can be retrieved. + // instantiated versions can be retrieved. // // If unspecified, retrieves the current version. - int32 version = 2; + int32 version = 2 [(google.api.field_behavior) = OPTIONAL]; } // A request to instantiate a workflow template. message InstantiateWorkflowTemplateRequest { - // Required. The "resource name" of the workflow template, as described - // in https://cloud.google.com/apis/design/resource_names of the form - // `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}` - string name = 1; + // Required. The resource name of the workflow template, as described + // in https://cloud.google.com/apis/design/resource_names. + // + // * For `projects.regions.workflowTemplates.instantiate`, the resource name + // of the template has the following format: + // `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}` + // + // * For `projects.locations.workflowTemplates.instantiate`, the resource name + // of the template has the following format: + // `projects/{project_id}/locations/{location}/workflowTemplates/{template_id}` + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "dataproc.googleapis.com/WorkflowTemplate" + } + ]; // Optional. The version of workflow template to instantiate. If specified, // the workflow will be instantiated only if the current version of @@ -571,7 +651,7 @@ message InstantiateWorkflowTemplateRequest { // // This option cannot be used to instantiate a previous version of // workflow template. - int32 version = 2; + int32 version = 2 [(google.api.field_behavior) = OPTIONAL]; // Optional. A tag that prevents multiple concurrent workflow // instances with the same tag from running. This mitigates risk of @@ -582,22 +662,34 @@ message InstantiateWorkflowTemplateRequest { // // The tag must contain only letters (a-z, A-Z), numbers (0-9), // underscores (_), and hyphens (-). The maximum length is 40 characters. - string request_id = 5; + string request_id = 5 [(google.api.field_behavior) = OPTIONAL]; // Optional. Map from parameter names to values that should be used for those // parameters. Values may not exceed 100 characters. - map parameters = 6; + map parameters = 6 [(google.api.field_behavior) = OPTIONAL]; } // A request to instantiate an inline workflow template. message InstantiateInlineWorkflowTemplateRequest { - // Required. The "resource name" of the workflow template region, as described - // in https://cloud.google.com/apis/design/resource_names of the form - // `projects/{project_id}/regions/{region}` - string parent = 1; + // Required. The resource name of the region or location, as described + // in https://cloud.google.com/apis/design/resource_names. + // + // * For `projects.regions.workflowTemplates,instantiateinline`, the resource + // name of the region has the following format: + // `projects/{project_id}/regions/{region}` + // + // * For `projects.locations.workflowTemplates.instantiateinline`, the + // resource name of the location has the following format: + // `projects/{project_id}/locations/{location}` + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + child_type: "dataproc.googleapis.com/WorkflowTemplate" + } + ]; // Required. The workflow template to instantiate. - WorkflowTemplate template = 2; + WorkflowTemplate template = 2 [(google.api.field_behavior) = REQUIRED]; // Optional. A tag that prevents multiple concurrent workflow // instances with the same tag from running. This mitigates risk of @@ -608,7 +700,7 @@ message InstantiateInlineWorkflowTemplateRequest { // // The tag must contain only letters (a-z, A-Z), numbers (0-9), // underscores (_), and hyphens (-). The maximum length is 40 characters. - string request_id = 3; + string request_id = 3 [(google.api.field_behavior) = OPTIONAL]; } // A request to update a workflow template. @@ -616,46 +708,75 @@ message UpdateWorkflowTemplateRequest { // Required. The updated workflow template. // // The `template.version` field must match the current version. - WorkflowTemplate template = 1; + WorkflowTemplate template = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "dataproc.googleapis.com/WorkflowTemplate" + } + ]; } // A request to list workflow templates in a project. message ListWorkflowTemplatesRequest { - // Required. The "resource name" of the region, as described - // in https://cloud.google.com/apis/design/resource_names of the form - // `projects/{project_id}/regions/{region}` - string parent = 1; + // Required. The resource name of the region or location, as described + // in https://cloud.google.com/apis/design/resource_names. + // + // * For `projects.regions.workflowTemplates,list`, the resource + // name of the region has the following format: + // `projects/{project_id}/regions/{region}` + // + // * For `projects.locations.workflowTemplates.list`, the + // resource name of the location has the following format: + // `projects/{project_id}/locations/{location}` + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + child_type: "dataproc.googleapis.com/WorkflowTemplate" + } + ]; // Optional. The maximum number of results to return in each response. - int32 page_size = 2; + int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL]; // Optional. The page token, returned by a previous call, to request the // next page of results. - string page_token = 3; + string page_token = 3 [(google.api.field_behavior) = OPTIONAL]; } // A response to a request to list workflow templates in a project. message ListWorkflowTemplatesResponse { // Output only. WorkflowTemplates list. - repeated WorkflowTemplate templates = 1; + repeated WorkflowTemplate templates = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. This token is included in the response if there are more // results to fetch. To fetch additional results, provide this value as the // page_token in a subsequent ListWorkflowTemplatesRequest. - string next_page_token = 2; + string next_page_token = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; } // A request to delete a workflow template. // // Currently started workflows will remain running. message DeleteWorkflowTemplateRequest { - // Required. The "resource name" of the workflow template, as described - // in https://cloud.google.com/apis/design/resource_names of the form - // `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}` - string name = 1; + // Required. The resource name of the workflow template, as described + // in https://cloud.google.com/apis/design/resource_names. + // + // * For `projects.regions.workflowTemplates.delete`, the resource name + // of the template has the following format: + // `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}` + // + // * For `projects.locations.workflowTemplates.instantiate`, the resource name + // of the template has the following format: + // `projects/{project_id}/locations/{location}/workflowTemplates/{template_id}` + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "dataproc.googleapis.com/WorkflowTemplate" + } + ]; // Optional. The version of workflow template to delete. If specified, // will only delete the template if the current server version matches // specified version. - int32 version = 2; + int32 version = 2 [(google.api.field_behavior) = OPTIONAL]; }