GlcoinCore/googleapis
1
0
Fork 0
mirror of https://github.com/googleapis/googleapis.git synced 2026-08-20 13:27:47 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Synchronize new proto/yaml changes.

Browse source 
PiperOrigin-RevId: 272971705
This commit is contained in:
Google APIs 2019-10-04 15:36:48 -07:00 • committed by Copybara-Service
parent 063d82fe6f
commit ceb8e2fb12
7 changed files with 1021 additions and 276 deletions
Download patch file Download diff file Expand all files Collapse all files

1
google/cloud/dataproc/v1/BUILD.bazel
View file

@ -9,6 +9,7 @@ load("@com_google_googleapis_imports//:imports.bzl", "proto_library_with_info")
proto_library(
name = "dataproc_proto",
srcs = [
"autoscaling_policies.proto",
"clusters.proto",
"jobs.proto",
"operations.proto",

340
google/cloud/dataproc/v1/autoscaling_policies.proto Normal file
View file

@ -0,0 +1,340 @@
// Copyright 2019 Google LLC.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
syntax = "proto3";
package google.cloud.dataproc.v1;
import "google/api/annotations.proto";
import "google/api/field_behavior.proto";
import "google/api/resource.proto";
import "google/protobuf/duration.proto";
import "google/protobuf/empty.proto";
import "google/api/client.proto";
option go_package = "google.golang.org/genproto/googleapis/cloud/dataproc/v1;dataproc";
option java_multiple_files = true;
option java_outer_classname = "AutoscalingPoliciesProto";
option java_package = "com.google.cloud.dataproc.v1";
// The API interface for managing autoscaling policies in the
// Google Cloud Dataproc API.
service AutoscalingPolicyService {
option (google.api.default_host) = "dataproc.googleapis.com";
option (google.api.oauth_scopes) = "https://www.googleapis.com/auth/cloud-platform";
// Creates new autoscaling policy.
rpc CreateAutoscalingPolicy(CreateAutoscalingPolicyRequest) returns (AutoscalingPolicy) {
option (google.api.http) = {
post: "/v1/{parent=projects/*/locations/*}/autoscalingPolicies"
body: "policy"
additional_bindings {
post: "/v1/{parent=projects/*/regions/*}/autoscalingPolicies"
body: "policy"
}
};
}
// Updates (replaces) autoscaling policy.
//
// Disabled check for update_mask, because all updates will be full
// replacements.
rpc UpdateAutoscalingPolicy(UpdateAutoscalingPolicyRequest) returns (AutoscalingPolicy) {
option (google.api.http) = {
put: "/v1/{policy.name=projects/*/locations/*/autoscalingPolicies/*}"
body: "policy"
additional_bindings {
put: "/v1/{policy.name=projects/*/regions/*/autoscalingPolicies/*}"
body: "policy"
}
};
}
// Retrieves autoscaling policy.
rpc GetAutoscalingPolicy(GetAutoscalingPolicyRequest) returns (AutoscalingPolicy) {
option (google.api.http) = {
get: "/v1/{name=projects/*/locations/*/autoscalingPolicies/*}"
additional_bindings {
get: "/v1/{name=projects/*/regions/*/autoscalingPolicies/*}"
}
};
}
// Lists autoscaling policies in the project.
rpc ListAutoscalingPolicies(ListAutoscalingPoliciesRequest) returns (ListAutoscalingPoliciesResponse) {
option (google.api.http) = {
get: "/v1/{parent=projects/*/locations/*}/autoscalingPolicies"
additional_bindings {
get: "/v1/{parent=projects/*/regions/*}/autoscalingPolicies"
}
};
}
// Deletes an autoscaling policy. It is an error to delete an autoscaling
// policy that is in use by one or more clusters.
rpc DeleteAutoscalingPolicy(DeleteAutoscalingPolicyRequest) returns (google.protobuf.Empty) {
option (google.api.http) = {
delete: "/v1/{name=projects/*/locations/*/autoscalingPolicies/*}"
additional_bindings {
delete: "/v1/{name=projects/*/regions/*/autoscalingPolicies/*}"
}
};
}
}
// Describes an autoscaling policy for Dataproc cluster autoscaler.
message AutoscalingPolicy {
option (google.api.resource) = {
type: "dataproc.googleapis.com/AutoscalingPolicy"
pattern: "projects/{project}/regions/{region}/autoscalingPolicies/{autoscaling_policy}"
};
// Required. The policy id.
//
// The id must contain only letters (a-z, A-Z), numbers (0-9),
// underscores (_), and hyphens (-). Cannot begin or end with underscore
// or hyphen. Must consist of between 3 and 50 characters.
//
string id = 1;
// Output only. The "resource name" of the autoscaling policy, as described
// in https://cloud.google.com/apis/design/resource_names.
//
// * For `projects.regions.autoscalingPolicies`, the resource name of the
// policy has the following format:
// `projects/{project_id}/regions/{region}/autoscalingPolicies/{policy_id}`
//
// * For `projects.locations.autoscalingPolicies`, the resource name of the
// policy has the following format:
// `projects/{project_id}/locations/{location}/autoscalingPolicies/{policy_id}`
string name = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
// Autoscaling algorithm for policy.
oneof algorithm {
BasicAutoscalingAlgorithm basic_algorithm = 3 [(google.api.field_behavior) = REQUIRED];
}
// Required. Describes how the autoscaler will operate for primary workers.
InstanceGroupAutoscalingPolicyConfig worker_config = 4 [(google.api.field_behavior) = REQUIRED];
// Optional. Describes how the autoscaler will operate for secondary workers.
InstanceGroupAutoscalingPolicyConfig secondary_worker_config = 5 [(google.api.field_behavior) = OPTIONAL];
}
// Basic algorithm for autoscaling.
message BasicAutoscalingAlgorithm {
// Required. YARN autoscaling configuration.
BasicYarnAutoscalingConfig yarn_config = 1 [(google.api.field_behavior) = REQUIRED];
// Optional. Duration between scaling events. A scaling period starts after
// the update operation from the previous event has completed.
//
// Bounds: [2m, 1d]. Default: 2m.
google.protobuf.Duration cooldown_period = 2 [(google.api.field_behavior) = OPTIONAL];
}
// Basic autoscaling configurations for YARN.
message BasicYarnAutoscalingConfig {
// Required. Timeout for YARN graceful decommissioning of Node Managers.
// Specifies the duration to wait for jobs to complete before forcefully
// removing workers (and potentially interrupting jobs). Only applicable to
// downscaling operations.
//
// Bounds: [0s, 1d].
google.protobuf.Duration graceful_decommission_timeout = 5 [(google.api.field_behavior) = REQUIRED];
// Required. Fraction of average pending memory in the last cooldown period
// for which to add workers. A scale-up factor of 1.0 will result in scaling
// up so that there is no pending memory remaining after the update (more
// aggressive scaling). A scale-up factor closer to 0 will result in a smaller
// magnitude of scaling up (less aggressive scaling).
//
// Bounds: [0.0, 1.0].
double scale_up_factor = 1 [(google.api.field_behavior) = REQUIRED];
// Required. Fraction of average pending memory in the last cooldown period
// for which to remove workers. A scale-down factor of 1 will result in
// scaling down so that there is no available memory remaining after the
// update (more aggressive scaling). A scale-down factor of 0 disables
// removing workers, which can be beneficial for autoscaling a single job.
//
// Bounds: [0.0, 1.0].
double scale_down_factor = 2 [(google.api.field_behavior) = REQUIRED];
// Optional. Minimum scale-up threshold as a fraction of total cluster size
// before scaling occurs. For example, in a 20-worker cluster, a threshold of
// 0.1 means the autoscaler must recommend at least a 2-worker scale-up for
// the cluster to scale. A threshold of 0 means the autoscaler will scale up
// on any recommended change.
//
// Bounds: [0.0, 1.0]. Default: 0.0.
double scale_up_min_worker_fraction = 3 [(google.api.field_behavior) = OPTIONAL];
// Optional. Minimum scale-down threshold as a fraction of total cluster size
// before scaling occurs. For example, in a 20-worker cluster, a threshold of
// 0.1 means the autoscaler must recommend at least a 2 worker scale-down for
// the cluster to scale. A threshold of 0 means the autoscaler will scale down
// on any recommended change.
//
// Bounds: [0.0, 1.0]. Default: 0.0.
double scale_down_min_worker_fraction = 4 [(google.api.field_behavior) = OPTIONAL];
}
// Configuration for the size bounds of an instance group, including its
// proportional size to other groups.
message InstanceGroupAutoscalingPolicyConfig {
// Optional. Minimum number of instances for this group.
//
// Primary workers - Bounds: [2, max_instances]. Default: 2.
// Secondary workers - Bounds: [0, max_instances]. Default: 0.
int32 min_instances = 1 [(google.api.field_behavior) = OPTIONAL];
// Required. Maximum number of instances for this group. Required for primary
// workers. Note that by default, clusters will not use secondary workers.
// Required for secondary workers if the minimum secondary instances is set.
//
// Primary workers - Bounds: [min_instances, ).
// Secondary workers - Bounds: [min_instances, ). Default: 0.
int32 max_instances = 2 [(google.api.field_behavior) = REQUIRED];
// Optional. Weight for the instance group, which is used to determine the
// fraction of total workers in the cluster from this instance group.
// For example, if primary workers have weight 2, and secondary workers have
// weight 1, the cluster will have approximately 2 primary workers for each
// secondary worker.
//
// The cluster may not reach the specified balance if constrained
// by min/max bounds or other autoscaling settings. For example, if
// `max_instances` for secondary workers is 0, then only primary workers will
// be added. The cluster can also be out of balance when created.
//
// If weight is not set on any instance group, the cluster will default to
// equal weight for all groups: the cluster will attempt to maintain an equal
// number of workers in each group within the configured size bounds for each
// group. If weight is set for one group only, the cluster will default to
// zero weight on the unset group. For example if weight is set only on
// primary workers, the cluster will use primary workers only and no
// secondary workers.
int32 weight = 3 [(google.api.field_behavior) = OPTIONAL];
}
// A request to create an autoscaling policy.
message CreateAutoscalingPolicyRequest {
// Required. The "resource name" of the region or location, as described
// in https://cloud.google.com/apis/design/resource_names.
//
// * For `projects.regions.autoscalingPolicies.create`, the resource name
// of the region has the following format:
// `projects/{project_id}/regions/{region}`
//
// * For `projects.locations.autoscalingPolicies.create`, the resource name
// of the location has the following format:
// `projects/{project_id}/locations/{location}`
string parent = 1 [
(google.api.field_behavior) = REQUIRED,
(google.api.resource_reference) = {
child_type: "dataproc.googleapis.com/AutoscalingPolicy"
}
];
// The autoscaling policy to create.
AutoscalingPolicy policy = 2;
}
// A request to fetch an autoscaling policy.
message GetAutoscalingPolicyRequest {
// Required. The "resource name" of the autoscaling policy, as described
// in https://cloud.google.com/apis/design/resource_names.
//
// * For `projects.regions.autoscalingPolicies.get`, the resource name
// of the policy has the following format:
// `projects/{project_id}/regions/{region}/autoscalingPolicies/{policy_id}`
//
// * For `projects.locations.autoscalingPolicies.get`, the resource name
// of the policy has the following format:
// `projects/{project_id}/locations/{location}/autoscalingPolicies/{policy_id}`
string name = 1 [
(google.api.field_behavior) = REQUIRED,
(google.api.resource_reference) = {
type: "dataproc.googleapis.com/AutoscalingPolicy"
}
];
}
// A request to update an autoscaling policy.
message UpdateAutoscalingPolicyRequest {
// Required. The updated autoscaling policy.
AutoscalingPolicy policy = 1 [(google.api.field_behavior) = REQUIRED];
}
// A request to delete an autoscaling policy.
//
// Autoscaling policies in use by one or more clusters will not be deleted.
message DeleteAutoscalingPolicyRequest {
// Required. The "resource name" of the autoscaling policy, as described
// in https://cloud.google.com/apis/design/resource_names.
//
// * For `projects.regions.autoscalingPolicies.delete`, the resource name
// of the policy has the following format:
// `projects/{project_id}/regions/{region}/autoscalingPolicies/{policy_id}`
//
// * For `projects.locations.autoscalingPolicies.delete`, the resource name
// of the policy has the following format:
// `projects/{project_id}/locations/{location}/autoscalingPolicies/{policy_id}`
string name = 1 [
(google.api.field_behavior) = REQUIRED,
(google.api.resource_reference) = {
type: "dataproc.googleapis.com/AutoscalingPolicy"
}
];
}
// A request to list autoscaling policies in a project.
message ListAutoscalingPoliciesRequest {
// Required. The "resource name" of the region or location, as described
// in https://cloud.google.com/apis/design/resource_names.
//
// * For `projects.regions.autoscalingPolicies.list`, the resource name
// of the region has the following format:
// `projects/{project_id}/regions/{region}`
//
// * For `projects.locations.autoscalingPolicies.list`, the resource name
// of the location has the following format:
// `projects/{project_id}/locations/{location}`
string parent = 1 [
(google.api.field_behavior) = REQUIRED,
(google.api.resource_reference) = {
child_type: "dataproc.googleapis.com/AutoscalingPolicy"
}
];
// Optional. The maximum number of results to return in each response.
// Must be less than or equal to 1000. Defaults to 100.
int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL];
// Optional. The page token, returned by a previous call, to request the
// next page of results.
string page_token = 3 [(google.api.field_behavior) = OPTIONAL];
}
// A response to a request to list autoscaling policies in a project.
message ListAutoscalingPoliciesResponse {
// Output only. Autoscaling policies list.
repeated AutoscalingPolicy policies = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. This token is included in the response if there are more
// results to fetch.
string next_page_token = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
}

307
google/cloud/dataproc/v1/clusters.proto
View file

@ -18,7 +18,8 @@ syntax = "proto3";
package google.cloud.dataproc.v1;
import "google/api/annotations.proto";
import "google/cloud/dataproc/v1/operations.proto";
import "google/api/client.proto";
import "google/api/field_behavior.proto";
import "google/cloud/dataproc/v1/shared.proto";
import "google/longrunning/operations.proto";
import "google/protobuf/duration.proto";
@ -33,27 +34,50 @@ option java_package = "com.google.cloud.dataproc.v1";
// The ClusterControllerService provides methods to manage clusters
// of Compute Engine instances.
service ClusterController {
// Creates a cluster in a project.
option (google.api.default_host) = "dataproc.googleapis.com";
option (google.api.oauth_scopes) = "https://www.googleapis.com/auth/cloud-platform";
// Creates a cluster in a project. The returned
// [Operation.metadata][google.longrunning.Operation.metadata] will be
// [ClusterOperationMetadata](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
rpc CreateCluster(CreateClusterRequest) returns (google.longrunning.Operation) {
option (google.api.http) = {
post: "/v1/projects/{project_id}/regions/{region}/clusters"
body: "cluster"
};
option (google.api.method_signature) = "project_id,region,cluster";
option (google.longrunning.operation_info) = {
response_type: "Cluster"
metadata_type: "google.cloud.dataproc.v1.ClusterOperationMetadata"
};
}
// Updates a cluster in a project.
// Updates a cluster in a project. The returned
// [Operation.metadata][google.longrunning.Operation.metadata] will be
// [ClusterOperationMetadata](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
rpc UpdateCluster(UpdateClusterRequest) returns (google.longrunning.Operation) {
option (google.api.http) = {
patch: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}"
body: "cluster"
};
option (google.longrunning.operation_info) = {
response_type: "Cluster"
metadata_type: "google.cloud.dataproc.v1.ClusterOperationMetadata"
};
}
// Deletes a cluster in a project.
// Deletes a cluster in a project. The returned
// [Operation.metadata][google.longrunning.Operation.metadata] will be
// [ClusterOperationMetadata](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
rpc DeleteCluster(DeleteClusterRequest) returns (google.longrunning.Operation) {
option (google.api.http) = {
delete: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}"
};
option (google.api.method_signature) = "project_id,region,cluster_name";
option (google.longrunning.operation_info) = {
response_type: "google.protobuf.Empty"
metadata_type: "google.cloud.dataproc.v1.ClusterOperationMetadata"
};
}
// Gets the resource representation for a cluster in a project.
@ -61,6 +85,7 @@ service ClusterController {
option (google.api.http) = {
get: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}"
};
option (google.api.method_signature) = "project_id,region,cluster_name";
}
// Lists all regions/{region}/clusters in a project.
@ -68,16 +93,27 @@ service ClusterController {
option (google.api.http) = {
get: "/v1/projects/{project_id}/regions/{region}/clusters"
};
option (google.api.method_signature) = "project_id,region";
option (google.api.method_signature) = "project_id,region,filter";
}
// Gets cluster diagnostic information.
// After the operation completes, the Operation.response field
// contains `DiagnoseClusterOutputLocation`.
// Gets cluster diagnostic information. The returned
// [Operation.metadata][google.longrunning.Operation.metadata] will be
// [ClusterOperationMetadata](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
// After the operation completes,
// [Operation.response][google.longrunning.Operation.response]
// contains
// [DiagnoseClusterResults](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#diagnoseclusterresults).
rpc DiagnoseCluster(DiagnoseClusterRequest) returns (google.longrunning.Operation) {
option (google.api.http) = {
post: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}:diagnose"
body: "*"
};
option (google.api.method_signature) = "project_id,region,cluster_name";
option (google.longrunning.operation_info) = {
response_type: "google.protobuf.Empty"
metadata_type: "DiagnoseClusterResults"
};
}
}
@ -102,17 +138,17 @@ message Cluster {
// characters, and must conform to [RFC
// 1035](https://www.ietf.org/rfc/rfc1035.txt). No more than 32 labels can be
// associated with a cluster.
map<string, string> labels = 8;
map<string, string> labels = 8 [(google.api.field_behavior) = OPTIONAL];
// Output only. Cluster status.
ClusterStatus status = 4;
ClusterStatus status = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. The previous cluster status.
repeated ClusterStatus status_history = 7;
repeated ClusterStatus status_history = 7 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. A cluster UUID (Unique Universal Identifier). Cloud Dataproc
// generates this value when it creates the cluster.
string cluster_uuid = 6;
string cluster_uuid = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
// Contains cluster daemon metrics such as HDFS and YARN stats.
//
@ -132,26 +168,26 @@ message ClusterConfig {
// and manage this project-level, per-location bucket (see
// [Cloud Dataproc staging
// bucket](/dataproc/docs/concepts/configuring-clusters/staging-bucket)).
string config_bucket = 1;
string config_bucket = 1 [(google.api.field_behavior) = OPTIONAL];
// Optional. The shared Compute Engine config settings for
// all instances in a cluster.
GceClusterConfig gce_cluster_config = 8;
GceClusterConfig gce_cluster_config = 8 [(google.api.field_behavior) = OPTIONAL];
// Optional. The Compute Engine config settings for
// the master instance in a cluster.
InstanceGroupConfig master_config = 9;
InstanceGroupConfig master_config = 9 [(google.api.field_behavior) = OPTIONAL];
// Optional. The Compute Engine config settings for
// worker instances in a cluster.
InstanceGroupConfig worker_config = 10;
InstanceGroupConfig worker_config = 10 [(google.api.field_behavior) = OPTIONAL];
// Optional. The Compute Engine config settings for
// additional worker instances in a cluster.
InstanceGroupConfig secondary_worker_config = 12;
InstanceGroupConfig secondary_worker_config = 12 [(google.api.field_behavior) = OPTIONAL];
// Optional. The config settings for software inside the cluster.
SoftwareConfig software_config = 13;
SoftwareConfig software_config = 13 [(google.api.field_behavior) = OPTIONAL];
// Optional. Commands to execute on each node after config is
// completed. By default, executables are run on master and all worker nodes.
@ -166,17 +202,38 @@ message ClusterConfig {
// else
// ... worker specific actions ...
// fi
repeated NodeInitializationAction initialization_actions = 11;
repeated NodeInitializationAction initialization_actions = 11 [(google.api.field_behavior) = OPTIONAL];
// Optional. Encryption settings for the cluster.
EncryptionConfig encryption_config = 15;
EncryptionConfig encryption_config = 15 [(google.api.field_behavior) = OPTIONAL];
// Optional. Autoscaling config for the policy associated with the cluster.
// Cluster does not autoscale if this field is unset.
AutoscalingConfig autoscaling_config = 18 [(google.api.field_behavior) = OPTIONAL];
// Optional. Security settings for the cluster.
SecurityConfig security_config = 16 [(google.api.field_behavior) = OPTIONAL];
}
// Autoscaling Policy config associated with the cluster.
message AutoscalingConfig {
// Optional. The autoscaling policy used by the cluster.
//
// Only resource names including projectid and location (region) are valid.
// Examples:
//
// * `https://www.googleapis.com/compute/v1/projects/[project_id]/locations/[dataproc_region]/autoscalingPolicies/[policy_id]`
// * `projects/[project_id]/locations/[dataproc_region]/autoscalingPolicies/[policy_id]`
//
// Note that the policy must be in the same project and Cloud Dataproc region.
string policy_uri = 1 [(google.api.field_behavior) = OPTIONAL];
}
// Encryption settings for the cluster.
message EncryptionConfig {
// Optional. The Cloud KMS key name to use for PD disk encryption for all
// instances in the cluster.
string gce_pd_kms_key_name = 1;
string gce_pd_kms_key_name = 1 [(google.api.field_behavior) = OPTIONAL];
}
// Common config settings for resources of Compute Engine cluster
@ -193,7 +250,7 @@ message GceClusterConfig {
// * `https://www.googleapis.com/compute/v1/projects/[project_id]/zones/[zone]`
// * `projects/[project_id]/zones/[zone]`
// * `us-central1-f`
string zone_uri = 1;
string zone_uri = 1 [(google.api.field_behavior) = OPTIONAL];
// Optional. The Compute Engine network to be used for machine
// communications. Cannot be specified with subnetwork_uri. If neither
@ -206,7 +263,7 @@ message GceClusterConfig {
// * `https://www.googleapis.com/compute/v1/projects/[project_id]/regions/global/default`
// * `projects/[project_id]/regions/global/default`
// * `default`
string network_uri = 2;
string network_uri = 2 [(google.api.field_behavior) = OPTIONAL];
// Optional. The Compute Engine subnetwork to be used for machine
// communications. Cannot be specified with network_uri.
@ -216,7 +273,7 @@ message GceClusterConfig {
// * `https://www.googleapis.com/compute/v1/projects/[project_id]/regions/us-east1/subnetworks/sub0`
// * `projects/[project_id]/regions/us-east1/subnetworks/sub0`
// * `sub0`
string subnetwork_uri = 6;
string subnetwork_uri = 6 [(google.api.field_behavior) = OPTIONAL];
// Optional. If true, all instances in the cluster will only have internal IP
// addresses. By default, clusters are not restricted to internal IP
@ -224,7 +281,7 @@ message GceClusterConfig {
// instance. This `internal_ip_only` restriction can only be enabled for
// subnetwork enabled networks, and all off-cluster dependencies must be
// configured to be accessible without external IP addresses.
bool internal_ip_only = 7;
bool internal_ip_only = 7 [(google.api.field_behavior) = OPTIONAL];
// Optional. The service account of the instances. Defaults to the default
// Compute Engine service account. Custom service accounts need
@ -237,7 +294,7 @@ message GceClusterConfig {
// https://cloud.google.com/compute/docs/access/service-accounts#custom_service_accounts
// for more information).
// Example: `[account_id]@[project_id].iam.gserviceaccount.com`
string service_account = 8;
string service_account = 8 [(google.api.field_behavior) = OPTIONAL];
// Optional. The URIs of service account scopes to be included in
// Compute Engine instances. The following base set of scopes is always
@ -253,7 +310,7 @@ message GceClusterConfig {
// * https://www.googleapis.com/auth/bigtable.admin.table
// * https://www.googleapis.com/auth/bigtable.data
// * https://www.googleapis.com/auth/devstorage.full_control
repeated string service_account_scopes = 3;
repeated string service_account_scopes = 3 [(google.api.field_behavior) = OPTIONAL];
// The Compute Engine tags to add to all instances (see
// [Tagging instances](/compute/docs/label-or-tag-resources#tags)).
@ -270,16 +327,16 @@ message GceClusterConfig {
message InstanceGroupConfig {
// Optional. The number of VM instances in the instance group.
// For master instance groups, must be set to 1.
int32 num_instances = 1;
int32 num_instances = 1 [(google.api.field_behavior) = OPTIONAL];
// Output only. The list of instance names. Cloud Dataproc derives the names
// from `cluster_name`, `num_instances`, and the instance group.
repeated string instance_names = 2;
repeated string instance_names = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
// Optional. The Compute Engine image resource used for cluster
// instances. It can be specified or may be inferred from
// `SoftwareConfig.image_version`.
string image_uri = 3;
string image_uri = 3 [(google.api.field_behavior) = OPTIONAL];
// Optional. The Compute Engine machine type used for cluster instances.
//
@ -294,36 +351,38 @@ message InstanceGroupConfig {
// Placement](/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement)
// feature, you must use the short name of the machine type
// resource, for example, `n1-standard-2`.
string machine_type_uri = 4;
string machine_type_uri = 4 [(google.api.field_behavior) = OPTIONAL];
// Optional. Disk option config settings.
DiskConfig disk_config = 5;
DiskConfig disk_config = 5 [(google.api.field_behavior) = OPTIONAL];
// Optional. Specifies that this instance group contains preemptible
// instances.
bool is_preemptible = 6;
bool is_preemptible = 6 [(google.api.field_behavior) = OPTIONAL];
// Output only. The config for Compute Engine Instance Group
// Manager that manages this group.
// This is only used for preemptible instance groups.
ManagedGroupConfig managed_group_config = 7;
ManagedGroupConfig managed_group_config = 7 [(google.api.field_behavior) = OUTPUT_ONLY];
// Optional. The Compute Engine accelerator configuration for these
// instances.
//
// **Beta Feature**: This feature is still under development. It may be
// changed before final release.
repeated AcceleratorConfig accelerators = 8;
repeated AcceleratorConfig accelerators = 8 [(google.api.field_behavior) = OPTIONAL];
// Optional. Specifies the minimum cpu platform for the Instance Group.
// See [Cloud Dataproc&rarr;Minimum CPU Platform]
// (/dataproc/docs/concepts/compute/dataproc-min-cpu).
string min_cpu_platform = 9 [(google.api.field_behavior) = OPTIONAL];
}
// Specifies the resources used to actively manage an instance group.
message ManagedGroupConfig {
// Output only. The name of the Instance Template used for the Managed
// Instance Group.
string instance_template_name = 1;
string instance_template_name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. The name of the Instance Group Manager for this group.
string instance_group_manager_name = 2;
string instance_group_manager_name = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
}
// Specifies the type and number of accelerator cards attached to the instances
@ -356,10 +415,10 @@ message DiskConfig {
// Optional. Type of the boot disk (default is "pd-standard").
// Valid values: "pd-ssd" (Persistent Disk Solid State Drive) or
// "pd-standard" (Persistent Disk Hard Disk Drive).
string boot_disk_type = 3;
string boot_disk_type = 3 [(google.api.field_behavior) = OPTIONAL];
// Optional. Size in GB of the boot disk (default is 500GB).
int32 boot_disk_size_gb = 1;
int32 boot_disk_size_gb = 1 [(google.api.field_behavior) = OPTIONAL];
// Optional. Number of attached SSDs, from 0 to 4 (default is 0).
// If SSDs are not attached, the boot disk is used to store runtime logs and
@ -367,20 +426,20 @@ message DiskConfig {
// If one or more SSDs are attached, this runtime bulk
// data is spread across them, and the boot disk contains only basic
// config and installed binaries.
int32 num_local_ssds = 2;
int32 num_local_ssds = 2 [(google.api.field_behavior) = OPTIONAL];
}
// Specifies an executable to run on a fully configured node and a
// timeout period for executable completion.
message NodeInitializationAction {
// Required. Cloud Storage URI of executable file.
string executable_file = 1;
string executable_file = 1 [(google.api.field_behavior) = REQUIRED];
// Optional. Amount of time executable has to complete. Default is
// 10 minutes. Cluster creation fails with an explanatory error message (the
// name of the executable that caused the error and the exceeded timeout
// period) if the executable is not completed at end of the timeout period.
google.protobuf.Duration execution_timeout = 2;
google.protobuf.Duration execution_timeout = 2 [(google.api.field_behavior) = OPTIONAL];
}
// The status of a cluster and its instances.
@ -426,17 +485,95 @@ message ClusterStatus {
}
// Output only. The cluster's state.
State state = 1;
State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. Optional details of cluster's state.
string detail = 2;
// Optional. Output only. Details of cluster's state.
string detail = 2 [
(google.api.field_behavior) = OUTPUT_ONLY,
(google.api.field_behavior) = OPTIONAL
];
// Output only. Time when this state was entered.
google.protobuf.Timestamp state_start_time = 3;
google.protobuf.Timestamp state_start_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. Additional state information that includes
// status reported by the agent.
Substate substate = 4;
Substate substate = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
}
// Security related configuration, including Kerberos.
message SecurityConfig {
// Kerberos related configuration.
KerberosConfig kerberos_config = 1;
}
// Specifies Kerberos related configuration.
message KerberosConfig {
// Optional. Flag to indicate whether to Kerberize the cluster.
bool enable_kerberos = 1 [(google.api.field_behavior) = OPTIONAL];
// Required. The Cloud Storage URI of a KMS encrypted file containing the root
// principal password.
string root_principal_password_uri = 2 [(google.api.field_behavior) = REQUIRED];
// Required. The uri of the KMS key used to encrypt various sensitive
// files.
string kms_key_uri = 3 [(google.api.field_behavior) = REQUIRED];
// Optional. The Cloud Storage URI of the keystore file used for SSL
// encryption. If not provided, Dataproc will provide a self-signed
// certificate.
string keystore_uri = 4 [(google.api.field_behavior) = OPTIONAL];
// Optional. The Cloud Storage URI of the truststore file used for SSL
// encryption. If not provided, Dataproc will provide a self-signed
// certificate.
string truststore_uri = 5 [(google.api.field_behavior) = OPTIONAL];
// Optional. The Cloud Storage URI of a KMS encrypted file containing the
// password to the user provided keystore. For the self-signed certificate,
// this password is generated by Dataproc.
string keystore_password_uri = 6 [(google.api.field_behavior) = OPTIONAL];
// Optional. The Cloud Storage URI of a KMS encrypted file containing the
// password to the user provided key. For the self-signed certificate, this
// password is generated by Dataproc.
string key_password_uri = 7 [(google.api.field_behavior) = OPTIONAL];
// Optional. The Cloud Storage URI of a KMS encrypted file containing the
// password to the user provided truststore. For the self-signed certificate,
// this password is generated by Dataproc.
string truststore_password_uri = 8 [(google.api.field_behavior) = OPTIONAL];
// Optional. The remote realm the Dataproc on-cluster KDC will trust, should
// the user enable cross realm trust.
string cross_realm_trust_realm = 9 [(google.api.field_behavior) = OPTIONAL];
// Optional. The KDC (IP or hostname) for the remote trusted realm in a cross
// realm trust relationship.
string cross_realm_trust_kdc = 10 [(google.api.field_behavior) = OPTIONAL];
// Optional. The admin server (IP or hostname) for the remote trusted realm in
// a cross realm trust relationship.
string cross_realm_trust_admin_server = 11 [(google.api.field_behavior) = OPTIONAL];
// Optional. The Cloud Storage URI of a KMS encrypted file containing the
// shared password between the on-cluster Kerberos realm and the remote
// trusted realm, in a cross realm trust relationship.
string cross_realm_trust_shared_password_uri = 12 [(google.api.field_behavior) = OPTIONAL];
// Optional. The Cloud Storage URI of a KMS encrypted file containing the
// master key of the KDC database.
string kdc_db_key_uri = 13 [(google.api.field_behavior) = OPTIONAL];
// Optional. The lifetime of the ticket granting ticket, in hours.
// If not specified, or user specifies 0, then default value 10
// will be used.
int32 tgt_lifetime_hours = 14 [(google.api.field_behavior) = OPTIONAL];
// Optional. The name of the on-cluster Kerberos realm.
// If not specified, the uppercased domain of hostnames will be the realm.
string realm = 15 [(google.api.field_behavior) = OPTIONAL];
}
// Specifies the selection and config of software inside the cluster.
@ -448,7 +585,7 @@ message SoftwareConfig {
// ["preview"
// version](/dataproc/docs/concepts/versioning/dataproc-versions#other_versions).
// If unspecified, it defaults to the latest Debian version.
string image_version = 1;
string image_version = 1 [(google.api.field_behavior) = OPTIONAL];
// Optional. The properties to set on daemon config files.
//
@ -468,10 +605,10 @@ message SoftwareConfig {
//
// For more information, see
// [Cluster properties](/dataproc/docs/concepts/cluster-properties).
map<string, string> properties = 2;
map<string, string> properties = 2 [(google.api.field_behavior) = OPTIONAL];
// The set of optional components to activate on the cluster.
repeated Component optional_components = 3;
// Optional. The set of components to activate on the cluster.
repeated Component optional_components = 3 [(google.api.field_behavior) = OPTIONAL];
}
// Contains cluster daemon metrics, such as HDFS and YARN stats.
@ -490,13 +627,13 @@ message ClusterMetrics {
message CreateClusterRequest {
// Required. The ID of the Google Cloud Platform project that the cluster
// belongs to.
string project_id = 1;
string project_id = 1 [(google.api.field_behavior) = REQUIRED];
// Required. The Cloud Dataproc region in which to handle the request.
string region = 3;
string region = 3 [(google.api.field_behavior) = REQUIRED];
// Required. The cluster to create.
Cluster cluster = 2;
Cluster cluster = 2 [(google.api.field_behavior) = REQUIRED];
// Optional. A unique id used to identify the request. If the server
// receives two [CreateClusterRequest][google.cloud.dataproc.v1.CreateClusterRequest] requests with the same
@ -509,23 +646,23 @@ message CreateClusterRequest {
//
// The id must contain only letters (a-z, A-Z), numbers (0-9),
// underscores (_), and hyphens (-). The maximum length is 40 characters.
string request_id = 4;
string request_id = 4 [(google.api.field_behavior) = OPTIONAL];
}
// A request to update a cluster.
message UpdateClusterRequest {
// Required. The ID of the Google Cloud Platform project the
// cluster belongs to.
string project_id = 1;
string project_id = 1 [(google.api.field_behavior) = REQUIRED];
// Required. The Cloud Dataproc region in which to handle the request.
string region = 5;
string region = 5 [(google.api.field_behavior) = REQUIRED];
// Required. The cluster name.
string cluster_name = 2;
string cluster_name = 2 [(google.api.field_behavior) = REQUIRED];
// Required. The changes to the cluster.
Cluster cluster = 3;
Cluster cluster = 3 [(google.api.field_behavior) = REQUIRED];
// Optional. Timeout for graceful YARN decomissioning. Graceful
// decommissioning allows removing nodes from the cluster without
@ -535,7 +672,7 @@ message UpdateClusterRequest {
// the maximum allowed timeout is 1 day.
//
// Only supported on Dataproc image versions 1.2 and higher.
google.protobuf.Duration graceful_decommission_timeout = 6;
google.protobuf.Duration graceful_decommission_timeout = 6 [(google.api.field_behavior) = OPTIONAL];
// Required. Specifies the path, relative to `Cluster`, of
// the field to update. For example, to change the number of workers
@ -582,9 +719,13 @@ message UpdateClusterRequest {
// <td><strong><em>config.secondary_worker_config.num_instances</em></strong></td>
// <td>Resize secondary worker group</td>
// </tr>
// <tr>
// <td>config.autoscaling_config.policy_uri</td><td>Use, stop using, or
// change autoscaling policies</td>
// </tr>
// </tbody>
// </table>
google.protobuf.FieldMask update_mask = 4;
google.protobuf.FieldMask update_mask = 4 [(google.api.field_behavior) = REQUIRED];
// Optional. A unique id used to identify the request. If the server
// receives two [UpdateClusterRequest][google.cloud.dataproc.v1.UpdateClusterRequest] requests with the same
@ -597,24 +738,24 @@ message UpdateClusterRequest {
//
// The id must contain only letters (a-z, A-Z), numbers (0-9),
// underscores (_), and hyphens (-). The maximum length is 40 characters.
string request_id = 7;
string request_id = 7 [(google.api.field_behavior) = OPTIONAL];
}
// A request to delete a cluster.
message DeleteClusterRequest {
// Required. The ID of the Google Cloud Platform project that the cluster
// belongs to.
string project_id = 1;
string project_id = 1 [(google.api.field_behavior) = REQUIRED];
// Required. The Cloud Dataproc region in which to handle the request.
string region = 3;
string region = 3 [(google.api.field_behavior) = REQUIRED];
// Required. The cluster name.
string cluster_name = 2;
string cluster_name = 2 [(google.api.field_behavior) = REQUIRED];
// Optional. Specifying the `cluster_uuid` means the RPC should fail
// (with error NOT_FOUND) if cluster with specified UUID does not exist.
string cluster_uuid = 4;
string cluster_uuid = 4 [(google.api.field_behavior) = OPTIONAL];
// Optional. A unique id used to identify the request. If the server
// receives two [DeleteClusterRequest][google.cloud.dataproc.v1.DeleteClusterRequest] requests with the same
@ -627,30 +768,30 @@ message DeleteClusterRequest {
//
// The id must contain only letters (a-z, A-Z), numbers (0-9),
// underscores (_), and hyphens (-). The maximum length is 40 characters.
string request_id = 5;
string request_id = 5 [(google.api.field_behavior) = OPTIONAL];
}
// Request to get the resource representation for a cluster in a project.
message GetClusterRequest {
// Required. The ID of the Google Cloud Platform project that the cluster
// belongs to.
string project_id = 1;
string project_id = 1 [(google.api.field_behavior) = REQUIRED];
// Required. The Cloud Dataproc region in which to handle the request.
string region = 3;
string region = 3 [(google.api.field_behavior) = REQUIRED];
// Required. The cluster name.
string cluster_name = 2;
string cluster_name = 2 [(google.api.field_behavior) = REQUIRED];
}
// A request to list the clusters in a project.
message ListClustersRequest {
// Required. The ID of the Google Cloud Platform project that the cluster
// belongs to.
string project_id = 1;
string project_id = 1 [(google.api.field_behavior) = REQUIRED];
// Required. The Cloud Dataproc region in which to handle the request.
string region = 4;
string region = 4 [(google.api.field_behavior) = REQUIRED];
// Optional. A filter constraining the clusters to list. Filters are
// case-sensitive and have the following syntax:
@ -671,37 +812,37 @@ message ListClustersRequest {
//
// status.state = ACTIVE AND clusterName = mycluster
// AND labels.env = staging AND labels.starred = *
string filter = 5;
string filter = 5 [(google.api.field_behavior) = OPTIONAL];
// Optional. The standard List page size.
int32 page_size = 2;
int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL];
// Optional. The standard List page token.
string page_token = 3;
string page_token = 3 [(google.api.field_behavior) = OPTIONAL];
}
// The list of all clusters in a project.
message ListClustersResponse {
// Output only. The clusters in the project.
repeated Cluster clusters = 1;
repeated Cluster clusters = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. This token is included in the response if there are more
// results to fetch. To fetch additional results, provide this value as the
// `page_token` in a subsequent `ListClustersRequest`.
string next_page_token = 2;
string next_page_token = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
}
// A request to collect cluster diagnostic information.
message DiagnoseClusterRequest {
// Required. The ID of the Google Cloud Platform project that the cluster
// belongs to.
string project_id = 1;
string project_id = 1 [(google.api.field_behavior) = REQUIRED];
// Required. The Cloud Dataproc region in which to handle the request.
string region = 3;
string region = 3 [(google.api.field_behavior) = REQUIRED];
// Required. The cluster name.
string cluster_name = 2;
string cluster_name = 2 [(google.api.field_behavior) = REQUIRED];
}
// The location of diagnostic output.
@ -709,5 +850,5 @@ message DiagnoseClusterResults {
// Output only. The Cloud Storage URI of the diagnostic output.
// The output report is a plain text file with a summary of collected
// diagnostics.
string output_uri = 1;
string output_uri = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
}

127
google/cloud/dataproc/v1/dataproc_v1.yaml Normal file
View file

@ -0,0 +1,127 @@
type: google.api.Service
config_version: 2
name: dataproc.googleapis.com
title: Cloud Dataproc API
apis:
- name: google.cloud.dataproc.v1.AutoscalingPolicyService
- name: google.cloud.dataproc.v1.ClusterController
- name: google.cloud.dataproc.v1.JobController
- name: google.cloud.dataproc.v1.WorkflowTemplateService
types:
- name: google.cloud.dataproc.v1.ClusterOperationMetadata
- name: google.cloud.dataproc.v1.DiagnoseClusterResults
- name: google.cloud.dataproc.v1.WorkflowMetadata
documentation:
summary: 'Manages Hadoop-based clusters and jobs on Google Cloud Platform.'
rules:
- selector: google.iam.v1.IAMPolicy.GetIamPolicy
description: |-
Gets the access control policy for a resource. Returns an empty policy
if the resource exists and does not have a policy set.
- selector: google.iam.v1.IAMPolicy.SetIamPolicy
description: |-
Sets the access control policy on the specified resource. Replaces
any existing policy.
Can return Public Errors: NOT_FOUND, INVALID_ARGUMENT and
PERMISSION_DENIED
- selector: google.iam.v1.IAMPolicy.TestIamPermissions
description: |-
Returns permissions that a caller has on the specified resource. If the
resource does not exist, this will return an empty set of
permissions, not a NOT_FOUND error.
Note: This operation is designed to be used for building
permission-aware UIs and command-line tools, not for authorization
checking. This operation may "fail open" without warning.
http:
rules:
- selector: google.iam.v1.IAMPolicy.GetIamPolicy
post: '/v1/{resource=projects/*/regions/*/clusters/*}:getIamPolicy'
body: '*'
additional_bindings:
- post: '/v1/{resource=projects/*/regions/*/jobs/*}:getIamPolicy'
body: '*'
- post: '/v1/{resource=projects/*/regions/*/operations/*}:getIamPolicy'
body: '*'
- post: '/v1/{resource=projects/*/regions/*/workflowTemplates/*}:getIamPolicy'
body: '*'
- post: '/v1/{resource=projects/*/locations/*/workflowTemplates/*}:getIamPolicy'
body: '*'
- post: '/v1/{resource=projects/*/regions/*/autoscalingPolicies/*}:getIamPolicy'
body: '*'
- post: '/v1/{resource=projects/*/locations/*/autoscalingPolicies/*}:getIamPolicy'
body: '*'
- selector: google.iam.v1.IAMPolicy.SetIamPolicy
post: '/v1/{resource=projects/*/regions/*/clusters/*}:setIamPolicy'
body: '*'
additional_bindings:
- post: '/v1/{resource=projects/*/regions/*/jobs/*}:setIamPolicy'
body: '*'
- post: '/v1/{resource=projects/*/regions/*/operations/*}:setIamPolicy'
body: '*'
- post: '/v1/{resource=projects/*/regions/*/workflowTemplates/*}:setIamPolicy'
body: '*'
- post: '/v1/{resource=projects/*/locations/*/workflowTemplates/*}:setIamPolicy'
body: '*'
- post: '/v1/{resource=projects/*/regions/*/autoscalingPolicies/*}:setIamPolicy'
body: '*'
- post: '/v1/{resource=projects/*/locations/*/autoscalingPolicies/*}:setIamPolicy'
body: '*'
- selector: google.iam.v1.IAMPolicy.TestIamPermissions
post: '/v1/{resource=projects/*/regions/*/clusters/*}:testIamPermissions'
body: '*'
additional_bindings:
- post: '/v1/{resource=projects/*/regions/*/jobs/*}:testIamPermissions'
body: '*'
- post: '/v1/{resource=projects/*/regions/*/operations/*}:testIamPermissions'
body: '*'
- post: '/v1/{resource=projects/*/regions/*/workflowTemplates/*}:testIamPermissions'
body: '*'
- post: '/v1/{resource=projects/*/locations/*/workflowTemplates/*}:testIamPermissions'
body: '*'
- post: '/v1/{resource=projects/*/regions/*/autoscalingPolicies/*}:testIamPermissions'
body: '*'
- post: '/v1/{resource=projects/*/locations/*/autoscalingPolicies/*}:testIamPermissions'
body: '*'
- selector: google.longrunning.Operations.CancelOperation
post: '/v1/{name=projects/*/regions/*/operations/*}:cancel'
- selector: google.longrunning.Operations.DeleteOperation
delete: '/v1/{name=projects/*/regions/*/operations/*}'
- selector: google.longrunning.Operations.GetOperation
get: '/v1/{name=projects/*/regions/*/operations/*}'
- selector: google.longrunning.Operations.ListOperations
get: '/v1/{name=projects/*/regions/*/operations}'
authentication:
rules:
- selector: 'google.cloud.dataproc.v1.AutoscalingPolicyService.*'
oauth:
canonical_scopes: |-
https://www.googleapis.com/auth/cloud-platform
- selector: 'google.cloud.dataproc.v1.ClusterController.*'
oauth:
canonical_scopes: |-
https://www.googleapis.com/auth/cloud-platform
- selector: 'google.cloud.dataproc.v1.JobController.*'
oauth:
canonical_scopes: |-
https://www.googleapis.com/auth/cloud-platform
- selector: 'google.cloud.dataproc.v1.WorkflowTemplateService.*'
oauth:
canonical_scopes: |-
https://www.googleapis.com/auth/cloud-platform
- selector: 'google.iam.v1.IAMPolicy.*'
oauth:
canonical_scopes: |-
https://www.googleapis.com/auth/cloud-platform
- selector: 'google.longrunning.Operations.*'
oauth:
canonical_scopes: |-
https://www.googleapis.com/auth/cloud-platform

184
google/cloud/dataproc/v1/jobs.proto
View file

@ -18,6 +18,8 @@ syntax = "proto3";
package google.cloud.dataproc.v1;
import "google/api/annotations.proto";
import "google/api/client.proto";
import "google/api/field_behavior.proto";
import "google/protobuf/empty.proto";
import "google/protobuf/field_mask.proto";
import "google/protobuf/timestamp.proto";
@ -29,12 +31,16 @@ option java_package = "com.google.cloud.dataproc.v1";
// The JobController provides methods to manage jobs.
service JobController {
option (google.api.default_host) = "dataproc.googleapis.com";
option (google.api.oauth_scopes) = "https://www.googleapis.com/auth/cloud-platform";
// Submits a job to a cluster.
rpc SubmitJob(SubmitJobRequest) returns (Job) {
option (google.api.http) = {
post: "/v1/projects/{project_id}/regions/{region}/jobs:submit"
body: "*"
};
option (google.api.method_signature) = "project_id,region,job";
}
// Gets the resource representation for a job in a project.
@ -42,6 +48,7 @@ service JobController {
option (google.api.http) = {
get: "/v1/projects/{project_id}/regions/{region}/jobs/{job_id}"
};
option (google.api.method_signature) = "project_id,region,job_id";
}
// Lists regions/{region}/jobs in a project.
@ -49,6 +56,8 @@ service JobController {
option (google.api.http) = {
get: "/v1/projects/{project_id}/regions/{region}/jobs"
};
option (google.api.method_signature) = "project_id,region";
option (google.api.method_signature) = "project_id,region,filter";
}
// Updates a job in a project.
@ -69,6 +78,7 @@ service JobController {
post: "/v1/projects/{project_id}/regions/{region}/jobs/{job_id}:cancel"
body: "*"
};
option (google.api.method_signature) = "project_id,region,job_id";
}
// Deletes the job from the project. If the job is active, the delete fails,
@ -77,6 +87,7 @@ service JobController {
option (google.api.http) = {
delete: "/v1/projects/{project_id}/regions/{region}/jobs/{job_id}"
};
option (google.api.method_signature) = "project_id,region,job_id";
}
}
@ -148,30 +159,30 @@ message HadoopJob {
// include arguments, such as `-libjars` or `-Dfoo=bar`, that can be set as
// job properties, since a collision may occur that causes an incorrect job
// submission.
repeated string args = 3;
repeated string args = 3 [(google.api.field_behavior) = OPTIONAL];
// Optional. Jar file URIs to add to the CLASSPATHs of the
// Hadoop driver and tasks.
repeated string jar_file_uris = 4;
repeated string jar_file_uris = 4 [(google.api.field_behavior) = OPTIONAL];
// Optional. HCFS (Hadoop Compatible Filesystem) URIs of files to be copied
// to the working directory of Hadoop drivers and distributed tasks. Useful
// for naively parallel tasks.
repeated string file_uris = 5;
repeated string file_uris = 5 [(google.api.field_behavior) = OPTIONAL];
// Optional. HCFS URIs of archives to be extracted in the working directory of
// Hadoop drivers and tasks. Supported file types:
// .jar, .tar, .tar.gz, .tgz, or .zip.
repeated string archive_uris = 6;
repeated string archive_uris = 6 [(google.api.field_behavior) = OPTIONAL];
// Optional. A mapping of property names to values, used to configure Hadoop.
// Properties that conflict with values set by the Cloud Dataproc API may be
// overwritten. Can include properties set in /etc/hadoop/conf/*-site and
// classes in user code.
map<string, string> properties = 7;
map<string, string> properties = 7 [(google.api.field_behavior) = OPTIONAL];
// Optional. The runtime log config for job execution.
LoggingConfig logging_config = 8;
LoggingConfig logging_config = 8 [(google.api.field_behavior) = OPTIONAL];
}
// A Cloud Dataproc job for running [Apache Spark](http://spark.apache.org/)
@ -194,29 +205,29 @@ message SparkJob {
// Optional. The arguments to pass to the driver. Do not include arguments,
// such as `--conf`, that can be set as job properties, since a collision may
// occur that causes an incorrect job submission.
repeated string args = 3;
repeated string args = 3 [(google.api.field_behavior) = OPTIONAL];
// Optional. HCFS URIs of jar files to add to the CLASSPATHs of the
// Spark driver and tasks.
repeated string jar_file_uris = 4;
repeated string jar_file_uris = 4 [(google.api.field_behavior) = OPTIONAL];
// Optional. HCFS URIs of files to be copied to the working directory of
// Spark drivers and distributed tasks. Useful for naively parallel tasks.
repeated string file_uris = 5;
repeated string file_uris = 5 [(google.api.field_behavior) = OPTIONAL];
// Optional. HCFS URIs of archives to be extracted in the working directory
// of Spark drivers and tasks. Supported file types:
// .jar, .tar, .tar.gz, .tgz, and .zip.
repeated string archive_uris = 6;
repeated string archive_uris = 6 [(google.api.field_behavior) = OPTIONAL];
// Optional. A mapping of property names to values, used to configure Spark.
// Properties that conflict with values set by the Cloud Dataproc API may be
// overwritten. Can include properties set in
// /etc/spark/conf/spark-defaults.conf and classes in user code.
map<string, string> properties = 7;
map<string, string> properties = 7 [(google.api.field_behavior) = OPTIONAL];
// Optional. The runtime log config for job execution.
LoggingConfig logging_config = 8;
LoggingConfig logging_config = 8 [(google.api.field_behavior) = OPTIONAL];
}
// A Cloud Dataproc job for running
@ -226,37 +237,37 @@ message SparkJob {
message PySparkJob {
// Required. The HCFS URI of the main Python file to use as the driver. Must
// be a .py file.
string main_python_file_uri = 1;
string main_python_file_uri = 1 [(google.api.field_behavior) = REQUIRED];
// Optional. The arguments to pass to the driver. Do not include arguments,
// such as `--conf`, that can be set as job properties, since a collision may
// occur that causes an incorrect job submission.
repeated string args = 2;
repeated string args = 2 [(google.api.field_behavior) = OPTIONAL];
// Optional. HCFS file URIs of Python files to pass to the PySpark
// framework. Supported file types: .py, .egg, and .zip.
repeated string python_file_uris = 3;
repeated string python_file_uris = 3 [(google.api.field_behavior) = OPTIONAL];
// Optional. HCFS URIs of jar files to add to the CLASSPATHs of the
// Python driver and tasks.
repeated string jar_file_uris = 4;
repeated string jar_file_uris = 4 [(google.api.field_behavior) = OPTIONAL];
// Optional. HCFS URIs of files to be copied to the working directory of
// Python drivers and distributed tasks. Useful for naively parallel tasks.
repeated string file_uris = 5;
repeated string file_uris = 5 [(google.api.field_behavior) = OPTIONAL];
// Optional. HCFS URIs of archives to be extracted in the working directory of
// .jar, .tar, .tar.gz, .tgz, and .zip.
repeated string archive_uris = 6;
repeated string archive_uris = 6 [(google.api.field_behavior) = OPTIONAL];
// Optional. A mapping of property names to values, used to configure PySpark.
// Properties that conflict with values set by the Cloud Dataproc API may be
// overwritten. Can include properties set in
// /etc/spark/conf/spark-defaults.conf and classes in user code.
map<string, string> properties = 7;
map<string, string> properties = 7 [(google.api.field_behavior) = OPTIONAL];
// Optional. The runtime log config for job execution.
LoggingConfig logging_config = 8;
LoggingConfig logging_config = 8 [(google.api.field_behavior) = OPTIONAL];
}
// A list of queries to run on a cluster.
@ -275,7 +286,7 @@ message QueryList {
// ]
// }
// }
repeated string queries = 1;
repeated string queries = 1 [(google.api.field_behavior) = REQUIRED];
}
// A Cloud Dataproc job for running [Apache Hive](https://hive.apache.org/)
@ -294,22 +305,22 @@ message HiveJob {
// Optional. Whether to continue executing queries if a query fails.
// The default value is `false`. Setting to `true` can be useful when
// executing independent parallel queries.
bool continue_on_failure = 3;
bool continue_on_failure = 3 [(google.api.field_behavior) = OPTIONAL];
// Optional. Mapping of query variable names to values (equivalent to the
// Hive command: `SET name="value";`).
map<string, string> script_variables = 4;
map<string, string> script_variables = 4 [(google.api.field_behavior) = OPTIONAL];
// Optional. A mapping of property names and values, used to configure Hive.
// Properties that conflict with values set by the Cloud Dataproc API may be
// overwritten. Can include properties set in /etc/hadoop/conf/*-site.xml,
// /etc/hive/conf/hive-site.xml, and classes in user code.
map<string, string> properties = 5;
map<string, string> properties = 5 [(google.api.field_behavior) = OPTIONAL];
// Optional. HCFS URIs of jar files to add to the CLASSPATH of the
// Hive server and Hadoop MapReduce (MR) tasks. Can contain Hive SerDes
// and UDFs.
repeated string jar_file_uris = 6;
repeated string jar_file_uris = 6 [(google.api.field_behavior) = OPTIONAL];
}
// A Cloud Dataproc job for running [Apache Spark
@ -327,18 +338,18 @@ message SparkSqlJob {
// Optional. Mapping of query variable names to values (equivalent to the
// Spark SQL command: SET `name="value";`).
map<string, string> script_variables = 3;
map<string, string> script_variables = 3 [(google.api.field_behavior) = OPTIONAL];
// Optional. A mapping of property names to values, used to configure
// Spark SQL's SparkConf. Properties that conflict with values set by the
// Cloud Dataproc API may be overwritten.
map<string, string> properties = 4;
map<string, string> properties = 4 [(google.api.field_behavior) = OPTIONAL];
// Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.
repeated string jar_file_uris = 56;
repeated string jar_file_uris = 56 [(google.api.field_behavior) = OPTIONAL];
// Optional. The runtime log config for job execution.
LoggingConfig logging_config = 6;
LoggingConfig logging_config = 6 [(google.api.field_behavior) = OPTIONAL];
}
// A Cloud Dataproc job for running [Apache Pig](https://pig.apache.org/)
@ -357,34 +368,34 @@ message PigJob {
// Optional. Whether to continue executing queries if a query fails.
// The default value is `false`. Setting to `true` can be useful when
// executing independent parallel queries.
bool continue_on_failure = 3;
bool continue_on_failure = 3 [(google.api.field_behavior) = OPTIONAL];
// Optional. Mapping of query variable names to values (equivalent to the Pig
// command: `name=[value]`).
map<string, string> script_variables = 4;
map<string, string> script_variables = 4 [(google.api.field_behavior) = OPTIONAL];
// Optional. A mapping of property names to values, used to configure Pig.
// Properties that conflict with values set by the Cloud Dataproc API may be
// overwritten. Can include properties set in /etc/hadoop/conf/*-site.xml,
// /etc/pig/conf/pig.properties, and classes in user code.
map<string, string> properties = 5;
map<string, string> properties = 5 [(google.api.field_behavior) = OPTIONAL];
// Optional. HCFS URIs of jar files to add to the CLASSPATH of
// the Pig Client and Hadoop MapReduce (MR) tasks. Can contain Pig UDFs.
repeated string jar_file_uris = 6;
repeated string jar_file_uris = 6 [(google.api.field_behavior) = OPTIONAL];
// Optional. The runtime log config for job execution.
LoggingConfig logging_config = 7;
LoggingConfig logging_config = 7 [(google.api.field_behavior) = OPTIONAL];
}
// Cloud Dataproc job config.
message JobPlacement {
// Required. The name of the cluster where the job will be submitted.
string cluster_name = 1;
string cluster_name = 1 [(google.api.field_behavior) = REQUIRED];
// Output only. A cluster UUID generated by the Cloud Dataproc service when
// the job is submitted.
string cluster_uuid = 2;
string cluster_uuid = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
}
// Cloud Dataproc job status.
@ -453,25 +464,28 @@ message JobStatus {
}
// Output only. A state message specifying the overall job state.
State state = 1;
State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. Optional job state details, such as an error
// Optional. Output only. Job state details, such as an error
// description if the state is <code>ERROR</code>.
string details = 2;
string details = 2 [
(google.api.field_behavior) = OUTPUT_ONLY,
(google.api.field_behavior) = OPTIONAL
];
// Output only. The time when this state was entered.
google.protobuf.Timestamp state_start_time = 6;
google.protobuf.Timestamp state_start_time = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. Additional state information, which includes
// status reported by the agent.
Substate substate = 7;
Substate substate = 7 [(google.api.field_behavior) = OUTPUT_ONLY];
}
// Encapsulates the full scoping used to reference a job.
message JobReference {
// Required. The ID of the Google Cloud Platform project that the job
// belongs to.
string project_id = 1;
string project_id = 1 [(google.api.field_behavior) = REQUIRED];
// Optional. The job ID, which must be unique within the project.
//
@ -479,7 +493,7 @@ message JobReference {
// underscores (_), or hyphens (-). The maximum length is 100 characters.
//
// If not specified by the caller, the job ID will be provided by the server.
string job_id = 2;
string job_id = 2 [(google.api.field_behavior) = OPTIONAL];
}
// A YARN application created by a job. Application information is a subset of
@ -520,19 +534,19 @@ message YarnApplication {
}
// Required. The application name.
string name = 1;
string name = 1 [(google.api.field_behavior) = REQUIRED];
// Required. The application state.
State state = 2;
State state = 2 [(google.api.field_behavior) = REQUIRED];
// Required. The numerical progress of the application, from 1 to 100.
float progress = 3;
float progress = 3 [(google.api.field_behavior) = REQUIRED];
// Optional. The HTTP URL of the ApplicationMaster, HistoryServer, or
// TimelineServer that provides application-specific information. The URL uses
// the internal hostname, and requires a proxy server for resolution and,
// possibly, access.
string tracking_url = 4;
string tracking_url = 4 [(google.api.field_behavior) = OPTIONAL];
}
// A Cloud Dataproc job resource.
@ -541,11 +555,11 @@ message Job {
// obtain the equivalent REST path of the job resource. If this property
// is not specified when a job is created, the server generates a
// <code>job_id</code>.
JobReference reference = 1;
JobReference reference = 1 [(google.api.field_behavior) = OPTIONAL];
// Required. Job information, including how, when, and where to
// run the job.
JobPlacement placement = 2;
JobPlacement placement = 2 [(google.api.field_behavior) = REQUIRED];
// Required. The application/framework-specific portion of the job.
oneof type_job {
@ -571,25 +585,25 @@ message Job {
// Output only. The job status. Additional application-specific
// status information may be contained in the <code>type_job</code>
// and <code>yarn_applications</code> fields.
JobStatus status = 8;
JobStatus status = 8 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. The previous job status.
repeated JobStatus status_history = 13;
repeated JobStatus status_history = 13 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. The collection of YARN applications spun up by this job.
//
// **Beta** Feature: This report is available for testing purposes only. It
// may be changed before final release.
repeated YarnApplication yarn_applications = 9;
repeated YarnApplication yarn_applications = 9 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. A URI pointing to the location of the stdout of the job's
// driver program.
string driver_output_resource_uri = 17;
string driver_output_resource_uri = 17 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. If present, the location of miscellaneous control files
// which may be used as part of job setup and handling. If not present,
// control files may be placed in the same location as `driver_output_uri`.
string driver_control_files_uri = 15;
string driver_control_files_uri = 15 [(google.api.field_behavior) = OUTPUT_ONLY];
// Optional. The labels to associate with this job.
// Label **keys** must contain 1 to 63 characters, and must conform to
@ -598,15 +612,15 @@ message Job {
// characters, and must conform to [RFC
// 1035](https://www.ietf.org/rfc/rfc1035.txt). No more than 32 labels can be
// associated with a job.
map<string, string> labels = 18;
map<string, string> labels = 18 [(google.api.field_behavior) = OPTIONAL];
// Optional. Job scheduling configuration.
JobScheduling scheduling = 20;
JobScheduling scheduling = 20 [(google.api.field_behavior) = OPTIONAL];
// Output only. A UUID that uniquely identifies a job within the project
// over time. This is in contrast to a user-settable reference.job_id that
// may be reused over time.
string job_uuid = 22;
string job_uuid = 22 [(google.api.field_behavior) = OUTPUT_ONLY];
}
// Job scheduling options.
@ -619,20 +633,20 @@ message JobScheduling {
// 4 times within 10 minute window.
//
// Maximum value is 10.
int32 max_failures_per_hour = 1;
int32 max_failures_per_hour = 1 [(google.api.field_behavior) = OPTIONAL];
}
// A request to submit a job.
message SubmitJobRequest {
// Required. The ID of the Google Cloud Platform project that the job
// belongs to.
string project_id = 1;
string project_id = 1 [(google.api.field_behavior) = REQUIRED];
// Required. The Cloud Dataproc region in which to handle the request.
string region = 3;
string region = 3 [(google.api.field_behavior) = REQUIRED];
// Required. The job resource.
Job job = 2;
Job job = 2 [(google.api.field_behavior) = REQUIRED];
// Optional. A unique id used to identify the request. If the server
// receives two [SubmitJobRequest][google.cloud.dataproc.v1.SubmitJobRequest] requests with the same
@ -645,20 +659,20 @@ message SubmitJobRequest {
//
// The id must contain only letters (a-z, A-Z), numbers (0-9),
// underscores (_), and hyphens (-). The maximum length is 40 characters.
string request_id = 4;
string request_id = 4 [(google.api.field_behavior) = OPTIONAL];
}
// A request to get the resource representation for a job in a project.
message GetJobRequest {
// Required. The ID of the Google Cloud Platform project that the job
// belongs to.
string project_id = 1;
string project_id = 1 [(google.api.field_behavior) = REQUIRED];
// Required. The Cloud Dataproc region in which to handle the request.
string region = 3;
string region = 3 [(google.api.field_behavior) = REQUIRED];
// Required. The job ID.
string job_id = 2;
string job_id = 2 [(google.api.field_behavior) = REQUIRED];
}
// A request to list jobs in a project.
@ -678,27 +692,27 @@ message ListJobsRequest {
// Required. The ID of the Google Cloud Platform project that the job
// belongs to.
string project_id = 1;
string project_id = 1 [(google.api.field_behavior) = REQUIRED];
// Required. The Cloud Dataproc region in which to handle the request.
string region = 6;
string region = 6 [(google.api.field_behavior) = REQUIRED];
// Optional. The number of results to return in each response.
int32 page_size = 2;
int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL];
// Optional. The page token, returned by a previous call, to request the
// next page of results.
string page_token = 3;
string page_token = 3 [(google.api.field_behavior) = OPTIONAL];
// Optional. If set, the returned jobs list includes only jobs that were
// submitted to the named cluster.
string cluster_name = 4;
string cluster_name = 4 [(google.api.field_behavior) = OPTIONAL];
// Optional. Specifies enumerated categories of jobs to list.
// (default = match ALL jobs).
//
// If `filter` is provided, `jobStateMatcher` will be ignored.
JobStateMatcher job_state_matcher = 5;
JobStateMatcher job_state_matcher = 5 [(google.api.field_behavior) = OPTIONAL];
// Optional. A filter constraining the jobs to list. Filters are
// case-sensitive and have the following syntax:
@ -714,23 +728,23 @@ message ListJobsRequest {
// Example filter:
//
// status.state = ACTIVE AND labels.env = staging AND labels.starred = *
string filter = 7;
string filter = 7 [(google.api.field_behavior) = OPTIONAL];
}
// A request to update a job.
message UpdateJobRequest {
// Required. The ID of the Google Cloud Platform project that the job
// belongs to.
string project_id = 1;
string project_id = 1 [(google.api.field_behavior) = REQUIRED];
// Required. The Cloud Dataproc region in which to handle the request.
string region = 2;
string region = 2 [(google.api.field_behavior) = REQUIRED];
// Required. The job ID.
string job_id = 3;
string job_id = 3 [(google.api.field_behavior) = REQUIRED];
// Required. The changes to the job.
Job job = 4;
Job job = 4 [(google.api.field_behavior) = REQUIRED];
// Required. Specifies the path, relative to <code>Job</code>, of
// the field to update. For example, to update the labels of a Job the
@ -738,42 +752,42 @@ message UpdateJobRequest {
// <code>labels</code>, and the `PATCH` request body would specify the new
// value. <strong>Note:</strong> Currently, <code>labels</code> is the only
// field that can be updated.
google.protobuf.FieldMask update_mask = 5;
google.protobuf.FieldMask update_mask = 5 [(google.api.field_behavior) = REQUIRED];
}
// A list of jobs in a project.
message ListJobsResponse {
// Output only. Jobs list.
repeated Job jobs = 1;
repeated Job jobs = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
// Optional. This token is included in the response if there are more results
// to fetch. To fetch additional results, provide this value as the
// `page_token` in a subsequent <code>ListJobsRequest</code>.
string next_page_token = 2;
string next_page_token = 2 [(google.api.field_behavior) = OPTIONAL];
}
// A request to cancel a job.
message CancelJobRequest {
// Required. The ID of the Google Cloud Platform project that the job
// belongs to.
string project_id = 1;
string project_id = 1 [(google.api.field_behavior) = REQUIRED];
// Required. The Cloud Dataproc region in which to handle the request.
string region = 3;
string region = 3 [(google.api.field_behavior) = REQUIRED];
// Required. The job ID.
string job_id = 2;
string job_id = 2 [(google.api.field_behavior) = REQUIRED];
}
// A request to delete a job.
message DeleteJobRequest {
// Required. The ID of the Google Cloud Platform project that the job
// belongs to.
string project_id = 1;
string project_id = 1 [(google.api.field_behavior) = REQUIRED];
// Required. The Cloud Dataproc region in which to handle the request.
string region = 3;
string region = 3 [(google.api.field_behavior) = REQUIRED];
// Required. The job ID.
string job_id = 2;
string job_id = 2 [(google.api.field_behavior) = REQUIRED];
}

27
google/cloud/dataproc/v1/operations.proto
View file

@ -17,8 +17,9 @@ syntax = "proto3";
package google.cloud.dataproc.v1;
import "google/api/annotations.proto";
import "google/api/field_behavior.proto";
import "google/protobuf/timestamp.proto";
import "google/api/annotations.proto";
option go_package = "google.golang.org/genproto/googleapis/cloud/dataproc/v1;dataproc";
option java_multiple_files = true;
@ -43,41 +44,41 @@ message ClusterOperationStatus {
}
// Output only. A message containing the operation state.
State state = 1;
State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. A message containing the detailed operation state.
string inner_state = 2;
string inner_state = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. A message containing any operation metadata details.
string details = 3;
string details = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. The time this state was entered.
google.protobuf.Timestamp state_start_time = 4;
google.protobuf.Timestamp state_start_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
}
// Metadata describing the operation.
message ClusterOperationMetadata {
// Output only. Name of the cluster for the operation.
string cluster_name = 7;
string cluster_name = 7 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. Cluster UUID for the operation.
string cluster_uuid = 8;
string cluster_uuid = 8 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. Current operation status.
ClusterOperationStatus status = 9;
ClusterOperationStatus status = 9 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. The previous operation status.
repeated ClusterOperationStatus status_history = 10;
repeated ClusterOperationStatus status_history = 10 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. The operation type.
string operation_type = 11;
string operation_type = 11 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. Short description of operation.
string description = 12;
string description = 12 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. Labels associated with the operation
map<string, string> labels = 13;
map<string, string> labels = 13 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. Errors encountered during operation execution.
repeated string warnings = 14;
repeated string warnings = 14 [(google.api.field_behavior) = OUTPUT_ONLY];
}

311
google/cloud/dataproc/v1/workflow_templates.proto
View file

@ -18,6 +18,9 @@ syntax = "proto3";
package google.cloud.dataproc.v1;
import "google/api/annotations.proto";
import "google/api/client.proto";
import "google/api/field_behavior.proto";
import "google/api/resource.proto";
import "google/cloud/dataproc/v1/clusters.proto";
import "google/cloud/dataproc/v1/jobs.proto";
import "google/longrunning/operations.proto";
@ -32,6 +35,9 @@ option java_package = "com.google.cloud.dataproc.v1";
// The API interface for managing Workflow Templates in the
// Cloud Dataproc API.
service WorkflowTemplateService {
option (google.api.default_host) = "dataproc.googleapis.com";
option (google.api.oauth_scopes) = "https://www.googleapis.com/auth/cloud-platform";
// Creates new workflow template.
rpc CreateWorkflowTemplate(CreateWorkflowTemplateRequest) returns (WorkflowTemplate) {
option (google.api.http) = {
@ -42,6 +48,7 @@ service WorkflowTemplateService {
body: "template"
}
};
option (google.api.method_signature) = "parent,template";
}
// Retrieves the latest workflow template.
@ -55,6 +62,7 @@ service WorkflowTemplateService {
get: "/v1/{name=projects/*/regions/*/workflowTemplates/*}"
}
};
option (google.api.method_signature) = "name";
}
// Instantiates a template and begins execution.
@ -70,7 +78,9 @@ service WorkflowTemplateService {
// clusters to be deleted.
//
// The [Operation.metadata][google.longrunning.Operation.metadata] will be
// [WorkflowMetadata][google.cloud.dataproc.v1.WorkflowMetadata].
// [WorkflowMetadata](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#workflowmetadata).
// Also see [Using
// WorkflowMetadata](/dataproc/docs/concepts/workflows/debugging#using_workflowmetadata).
//
// On successful completion,
// [Operation.response][google.longrunning.Operation.response] will be
@ -84,6 +94,12 @@ service WorkflowTemplateService {
body: "*"
}
};
option (google.api.method_signature) = "name";
option (google.api.method_signature) = "name,parameters";
option (google.longrunning.operation_info) = {
response_type: "google.protobuf.Empty"
metadata_type: "WorkflowMetadata"
};
}
// Instantiates a template and begins execution.
@ -103,7 +119,9 @@ service WorkflowTemplateService {
// clusters to be deleted.
//
// The [Operation.metadata][google.longrunning.Operation.metadata] will be
// [WorkflowMetadata][google.cloud.dataproc.v1.WorkflowMetadata].
// [WorkflowMetadata](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#workflowmetadata).
// Also see [Using
// WorkflowMetadata](/dataproc/docs/concepts/workflows/debugging#using_workflowmetadata).
//
// On successful completion,
// [Operation.response][google.longrunning.Operation.response] will be
@ -117,6 +135,11 @@ service WorkflowTemplateService {
body: "template"
}
};
option (google.api.method_signature) = "parent,template";
option (google.longrunning.operation_info) = {
response_type: "google.protobuf.Empty"
metadata_type: "WorkflowMetadata"
};
}
// Updates (replaces) workflow template. The updated template
@ -130,6 +153,7 @@ service WorkflowTemplateService {
body: "template"
}
};
option (google.api.method_signature) = "template";
}
// Lists workflows that match the specified filter in the request.
@ -140,6 +164,7 @@ service WorkflowTemplateService {
get: "/v1/{parent=projects/*/regions/*}/workflowTemplates"
}
};
option (google.api.method_signature) = "parent";
}
// Deletes a workflow template. It does not cancel in-progress workflows.
@ -150,22 +175,32 @@ service WorkflowTemplateService {
delete: "/v1/{name=projects/*/regions/*/workflowTemplates/*}"
}
};
option (google.api.method_signature) = "name";
}
}
// A Cloud Dataproc workflow template resource.
message WorkflowTemplate {
// Required. The template id.
//
// The id must contain only letters (a-z, A-Z), numbers (0-9),
// underscores (_), and hyphens (-). Cannot begin or end with underscore
// or hyphen. Must consist of between 3 and 50 characters.
string id = 2;
option (google.api.resource) = {
type: "dataproc.googleapis.com/WorkflowTemplate"
pattern: "projects/{project}/regions/{region}/workflowTemplates/{workflow_template}"
pattern: "projects/{project}/locations/{location}/workflowTemplates/{workflow_template}"
history: ORIGINALLY_SINGLE_PATTERN
};
// Output only. The "resource name" of the template, as described
// in https://cloud.google.com/apis/design/resource_names of the form
// `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}`
string name = 1;
string id = 2 [(google.api.field_behavior) = REQUIRED];
// Output only. The resource name of the workflow template, as described
// in https://cloud.google.com/apis/design/resource_names.
//
// * For `projects.regions.workflowTemplates`, the resource name of the
// template has the following format:
// `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}`
//
// * For `projects.locations.workflowTemplates`, the resource name of the
// template has the following format:
// `projects/{project_id}/locations/{location}/workflowTemplates/{template_id}`
string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
// Optional. Used to perform a consistent read-modify-write.
//
@ -176,13 +211,13 @@ message WorkflowTemplate {
// the current template with the `version` field filled in with the
// current server version. The user updates other fields in the template,
// then returns it as part of the `UpdateWorkflowTemplate` request.
int32 version = 3;
int32 version = 3 [(google.api.field_behavior) = OPTIONAL];
// Output only. The time template was created.
google.protobuf.Timestamp create_time = 4;
google.protobuf.Timestamp create_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. The time template was last updated.
google.protobuf.Timestamp update_time = 5;
google.protobuf.Timestamp update_time = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
// Optional. The labels to associate with this template. These labels
// will be propagated to all jobs and clusters created by the workflow
@ -196,18 +231,18 @@ message WorkflowTemplate {
// [RFC 1035](https://www.ietf.org/rfc/rfc1035.txt).
//
// No more than 32 labels can be associated with a template.
map<string, string> labels = 6;
map<string, string> labels = 6 [(google.api.field_behavior) = OPTIONAL];
// Required. WorkflowTemplate scheduling information.
WorkflowTemplatePlacement placement = 7;
WorkflowTemplatePlacement placement = 7 [(google.api.field_behavior) = REQUIRED];
// Required. The Directed Acyclic Graph of Jobs to submit.
repeated OrderedJob jobs = 8;
repeated OrderedJob jobs = 8 [(google.api.field_behavior) = REQUIRED];
// Optional. Template parameters whose values are substituted into the
// Optional. emplate parameters whose values are substituted into the
// template. Values for parameters must be provided when the template is
// instantiated.
repeated TemplateParameter parameters = 9;
repeated TemplateParameter parameters = 9 [(google.api.field_behavior) = OPTIONAL];
}
// Specifies workflow execution target.
@ -217,7 +252,7 @@ message WorkflowTemplatePlacement {
// Required. Specifies where workflow executes; either on a managed
// cluster or an existing cluster chosen by labels.
oneof placement {
// Optional. A cluster that is managed by the workflow.
// A cluster that is managed by the workflow.
ManagedCluster managed_cluster = 1;
// Optional. A selector that chooses target cluster for jobs based
@ -236,10 +271,10 @@ message ManagedCluster {
// The name must contain only lower-case letters (a-z), numbers (0-9),
// and hyphens (-). Must begin with a letter. Cannot begin or end with
// hyphen. Must consist of between 2 and 35 characters.
string cluster_name = 2;
string cluster_name = 2 [(google.api.field_behavior) = REQUIRED];
// Required. The cluster configuration.
ClusterConfig config = 3;
ClusterConfig config = 3 [(google.api.field_behavior) = REQUIRED];
// Optional. The labels to associate with this cluster.
//
@ -251,7 +286,7 @@ message ManagedCluster {
// the following PCRE regular expression: [\p{Ll}\p{Lo}\p{N}_-]{0,63}
//
// No more than 32 labels can be associated with a given cluster.
map<string, string> labels = 4;
map<string, string> labels = 4 [(google.api.field_behavior) = OPTIONAL];
}
// A selector that chooses target cluster for jobs based on metadata.
@ -261,11 +296,11 @@ message ClusterSelector {
//
// If unspecified, the zone of the first cluster matching the selector
// is used.
string zone = 1;
string zone = 1 [(google.api.field_behavior) = OPTIONAL];
// Required. The cluster labels. Cluster must have all labels
// to match.
map<string, string> cluster_labels = 2;
map<string, string> cluster_labels = 2 [(google.api.field_behavior) = REQUIRED];
}
// A job executed by the workflow.
@ -281,7 +316,7 @@ message OrderedJob {
// The id must contain only letters (a-z, A-Z), numbers (0-9),
// underscores (_), and hyphens (-). Cannot begin or end with underscore
// or hyphen. Must consist of between 3 and 50 characters.
string step_id = 1;
string step_id = 1 [(google.api.field_behavior) = REQUIRED];
// Required. The job definition.
oneof job_type {
@ -314,14 +349,14 @@ message OrderedJob {
// the following regular expression: [\p{Ll}\p{Lo}\p{N}_-]{0,63}
//
// No more than 32 labels can be associated with a given job.
map<string, string> labels = 8;
map<string, string> labels = 8 [(google.api.field_behavior) = OPTIONAL];
// Optional. Job scheduling configuration.
JobScheduling scheduling = 9;
JobScheduling scheduling = 9 [(google.api.field_behavior) = OPTIONAL];
// Optional. The optional list of prerequisite job step_ids.
// If not specified, the job will start at the beginning of workflow.
repeated string prerequisite_step_ids = 10;
repeated string prerequisite_step_ids = 10 [(google.api.field_behavior) = OPTIONAL];
}
// A configurable parameter that replaces one or more fields in the template.
@ -334,14 +369,14 @@ message OrderedJob {
// - Main class (in HadoopJob and SparkJob)
// - Zone (in ClusterSelector)
message TemplateParameter {
// Required. Parameter name.
// Required. Parameter name.
// The parameter name is used as the key, and paired with the
// parameter value, which are passed to the template when the template
// is instantiated.
// The name must contain only capital letters (A-Z), numbers (0-9), and
// underscores (_), and must not start with a number. The maximum length is
// 40 characters.
string name = 1;
string name = 1 [(google.api.field_behavior) = REQUIRED];
// Required. Paths to all fields that the parameter replaces.
// A field is allowed to appear in at most one parameter's list of field
@ -387,14 +422,14 @@ message TemplateParameter {
//
// - placement.clusterSelector.clusterLabels
// - jobs['step-id'].sparkJob.args
repeated string fields = 2;
repeated string fields = 2 [(google.api.field_behavior) = REQUIRED];
// Optional. Brief description of the parameter.
// Must not exceed 1024 characters.
string description = 3;
string description = 3 [(google.api.field_behavior) = OPTIONAL];
// Optional. Validation rules to be applied to this parameter's value.
ParameterValidation validation = 4;
ParameterValidation validation = 4 [(google.api.field_behavior) = OPTIONAL];
}
// Configuration for parameter validation.
@ -414,13 +449,13 @@ message RegexValidation {
// Required. RE2 regular expressions used to validate the parameter's value.
// The value must match the regex in its entirety (substring
// matches are not sufficient).
repeated string regexes = 1;
repeated string regexes = 1 [(google.api.field_behavior) = REQUIRED];
}
// Validation based on a list of allowed values.
message ValueValidation {
// Required. List of allowed values for the parameter.
repeated string values = 1;
repeated string values = 1 [(google.api.field_behavior) = REQUIRED];
}
// A Cloud Dataproc workflow template resource.
@ -440,57 +475,66 @@ message WorkflowMetadata {
DONE = 3;
}
// Output only. The "resource name" of the template.
string template = 1;
// Output only. The resource name of the workflow template as described
// in https://cloud.google.com/apis/design/resource_names.
//
// * For `projects.regions.workflowTemplates`, the resource name of the
// template has the following format:
// `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}`
//
// * For `projects.locations.workflowTemplates`, the resource name of the
// template has the following format:
// `projects/{project_id}/locations/{location}/workflowTemplates/{template_id}`
string template = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. The version of template at the time of
// workflow instantiation.
int32 version = 2;
int32 version = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. The create cluster operation metadata.
ClusterOperation create_cluster = 3;
ClusterOperation create_cluster = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. The workflow graph.
WorkflowGraph graph = 4;
WorkflowGraph graph = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. The delete cluster operation metadata.
ClusterOperation delete_cluster = 5;
ClusterOperation delete_cluster = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. The workflow state.
State state = 6;
State state = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. The name of the target cluster.
string cluster_name = 7;
string cluster_name = 7 [(google.api.field_behavior) = OUTPUT_ONLY];
// Map from parameter names to values that were used for those parameters.
map<string, string> parameters = 8;
// Output only. Workflow start time.
google.protobuf.Timestamp start_time = 9;
google.protobuf.Timestamp start_time = 9 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. Workflow end time.
google.protobuf.Timestamp end_time = 10;
google.protobuf.Timestamp end_time = 10 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. The UUID of target cluster.
string cluster_uuid = 11;
string cluster_uuid = 11 [(google.api.field_behavior) = OUTPUT_ONLY];
}
// The cluster operation triggered by a workflow.
message ClusterOperation {
// Output only. The id of the cluster operation.
string operation_id = 1;
string operation_id = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. Error, if operation failed.
string error = 2;
string error = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. Indicates the operation is done.
bool done = 3;
bool done = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
}
// The workflow graph.
message WorkflowGraph {
// Output only. The workflow nodes.
repeated WorkflowNode nodes = 1;
repeated WorkflowNode nodes = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
}
// The workflow node.
@ -518,52 +562,88 @@ message WorkflowNode {
}
// Output only. The name of the node.
string step_id = 1;
string step_id = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. Node's prerequisite nodes.
repeated string prerequisite_step_ids = 2;
repeated string prerequisite_step_ids = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. The job id; populated after the node enters RUNNING state.
string job_id = 3;
string job_id = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. The node state.
NodeState state = 5;
NodeState state = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. The error detail.
string error = 6;
string error = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
}
// A request to create a workflow template.
message CreateWorkflowTemplateRequest {
// Required. The "resource name" of the region, as described
// in https://cloud.google.com/apis/design/resource_names of the form
// `projects/{project_id}/regions/{region}`
string parent = 1;
// Required. The resource name of the region or location, as described
// in https://cloud.google.com/apis/design/resource_names.
//
// * For `projects.regions.workflowTemplates,create`, the resource name of the
// region has the following format:
// `projects/{project_id}/regions/{region}`
//
// * For `projects.locations.workflowTemplates.create`, the resource name of
// the location has the following format:
// `projects/{project_id}/locations/{location}`
string parent = 1 [
(google.api.field_behavior) = REQUIRED,
(google.api.resource_reference) = {
child_type: "dataproc.googleapis.com/WorkflowTemplate"
}
];
// Required. The Dataproc workflow template to create.
WorkflowTemplate template = 2;
WorkflowTemplate template = 2 [(google.api.field_behavior) = REQUIRED];
}
// A request to fetch a workflow template.
message GetWorkflowTemplateRequest {
// Required. The "resource name" of the workflow template, as described
// in https://cloud.google.com/apis/design/resource_names of the form
// `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}`
string name = 1;
// Required. The resource name of the workflow template, as described
// in https://cloud.google.com/apis/design/resource_names.
//
// * For `projects.regions.workflowTemplates.get`, the resource name of the
// template has the following format:
// `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}`
//
// * For `projects.locations.workflowTemplates.get`, the resource name of the
// template has the following format:
// `projects/{project_id}/locations/{location}/workflowTemplates/{template_id}`
string name = 1 [
(google.api.field_behavior) = REQUIRED,
(google.api.resource_reference) = {
type: "dataproc.googleapis.com/WorkflowTemplate"
}
];
// Optional. The version of workflow template to retrieve. Only previously
// instatiated versions can be retrieved.
// instantiated versions can be retrieved.
//
// If unspecified, retrieves the current version.
int32 version = 2;
int32 version = 2 [(google.api.field_behavior) = OPTIONAL];
}
// A request to instantiate a workflow template.
message InstantiateWorkflowTemplateRequest {
// Required. The "resource name" of the workflow template, as described
// in https://cloud.google.com/apis/design/resource_names of the form
// `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}`
string name = 1;
// Required. The resource name of the workflow template, as described
// in https://cloud.google.com/apis/design/resource_names.
//
// * For `projects.regions.workflowTemplates.instantiate`, the resource name
// of the template has the following format:
// `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}`
//
// * For `projects.locations.workflowTemplates.instantiate`, the resource name
// of the template has the following format:
// `projects/{project_id}/locations/{location}/workflowTemplates/{template_id}`
string name = 1 [
(google.api.field_behavior) = REQUIRED,
(google.api.resource_reference) = {
type: "dataproc.googleapis.com/WorkflowTemplate"
}
];
// Optional. The version of workflow template to instantiate. If specified,
// the workflow will be instantiated only if the current version of
@ -571,7 +651,7 @@ message InstantiateWorkflowTemplateRequest {
//
// This option cannot be used to instantiate a previous version of
// workflow template.
int32 version = 2;
int32 version = 2 [(google.api.field_behavior) = OPTIONAL];
// Optional. A tag that prevents multiple concurrent workflow
// instances with the same tag from running. This mitigates risk of
@ -582,22 +662,34 @@ message InstantiateWorkflowTemplateRequest {
//
// The tag must contain only letters (a-z, A-Z), numbers (0-9),
// underscores (_), and hyphens (-). The maximum length is 40 characters.
string request_id = 5;
string request_id = 5 [(google.api.field_behavior) = OPTIONAL];
// Optional. Map from parameter names to values that should be used for those
// parameters. Values may not exceed 100 characters.
map<string, string> parameters = 6;
map<string, string> parameters = 6 [(google.api.field_behavior) = OPTIONAL];
}
// A request to instantiate an inline workflow template.
message InstantiateInlineWorkflowTemplateRequest {
// Required. The "resource name" of the workflow template region, as described
// in https://cloud.google.com/apis/design/resource_names of the form
// `projects/{project_id}/regions/{region}`
string parent = 1;
// Required. The resource name of the region or location, as described
// in https://cloud.google.com/apis/design/resource_names.
//
// * For `projects.regions.workflowTemplates,instantiateinline`, the resource
// name of the region has the following format:
// `projects/{project_id}/regions/{region}`
//
// * For `projects.locations.workflowTemplates.instantiateinline`, the
// resource name of the location has the following format:
// `projects/{project_id}/locations/{location}`
string parent = 1 [
(google.api.field_behavior) = REQUIRED,
(google.api.resource_reference) = {
child_type: "dataproc.googleapis.com/WorkflowTemplate"
}
];
// Required. The workflow template to instantiate.
WorkflowTemplate template = 2;
WorkflowTemplate template = 2 [(google.api.field_behavior) = REQUIRED];
// Optional. A tag that prevents multiple concurrent workflow
// instances with the same tag from running. This mitigates risk of
@ -608,7 +700,7 @@ message InstantiateInlineWorkflowTemplateRequest {
//
// The tag must contain only letters (a-z, A-Z), numbers (0-9),
// underscores (_), and hyphens (-). The maximum length is 40 characters.
string request_id = 3;
string request_id = 3 [(google.api.field_behavior) = OPTIONAL];
}
// A request to update a workflow template.
@ -616,46 +708,75 @@ message UpdateWorkflowTemplateRequest {
// Required. The updated workflow template.
//
// The `template.version` field must match the current version.
WorkflowTemplate template = 1;
WorkflowTemplate template = 1 [
(google.api.field_behavior) = REQUIRED,
(google.api.resource_reference) = {
type: "dataproc.googleapis.com/WorkflowTemplate"
}
];
}
// A request to list workflow templates in a project.
message ListWorkflowTemplatesRequest {
// Required. The "resource name" of the region, as described
// in https://cloud.google.com/apis/design/resource_names of the form
// `projects/{project_id}/regions/{region}`
string parent = 1;
// Required. The resource name of the region or location, as described
// in https://cloud.google.com/apis/design/resource_names.
//
// * For `projects.regions.workflowTemplates,list`, the resource
// name of the region has the following format:
// `projects/{project_id}/regions/{region}`
//
// * For `projects.locations.workflowTemplates.list`, the
// resource name of the location has the following format:
// `projects/{project_id}/locations/{location}`
string parent = 1 [
(google.api.field_behavior) = REQUIRED,
(google.api.resource_reference) = {
child_type: "dataproc.googleapis.com/WorkflowTemplate"
}
];
// Optional. The maximum number of results to return in each response.
int32 page_size = 2;
int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL];
// Optional. The page token, returned by a previous call, to request the
// next page of results.
string page_token = 3;
string page_token = 3 [(google.api.field_behavior) = OPTIONAL];
}
// A response to a request to list workflow templates in a project.
message ListWorkflowTemplatesResponse {
// Output only. WorkflowTemplates list.
repeated WorkflowTemplate templates = 1;
repeated WorkflowTemplate templates = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. This token is included in the response if there are more
// results to fetch. To fetch additional results, provide this value as the
// page_token in a subsequent <code>ListWorkflowTemplatesRequest</code>.
string next_page_token = 2;
string next_page_token = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
}
// A request to delete a workflow template.
//
// Currently started workflows will remain running.
message DeleteWorkflowTemplateRequest {
// Required. The "resource name" of the workflow template, as described
// in https://cloud.google.com/apis/design/resource_names of the form
// `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}`
string name = 1;
// Required. The resource name of the workflow template, as described
// in https://cloud.google.com/apis/design/resource_names.
//
// * For `projects.regions.workflowTemplates.delete`, the resource name
// of the template has the following format:
// `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}`
//
// * For `projects.locations.workflowTemplates.instantiate`, the resource name
// of the template has the following format:
// `projects/{project_id}/locations/{location}/workflowTemplates/{template_id}`
string name = 1 [
(google.api.field_behavior) = REQUIRED,
(google.api.resource_reference) = {
type: "dataproc.googleapis.com/WorkflowTemplate"
}
];
// Optional. The version of workflow template to delete. If specified,
// will only delete the template if the current server version matches
// specified version.
int32 version = 2;
int32 version = 2 [(google.api.field_behavior) = OPTIONAL];
}