mirror of
https://github.com/googleapis/googleapis.git
synced 2026-08-16 13:00:34 +02:00
805 lines
28 KiB
Protocol Buffer
805 lines
28 KiB
Protocol Buffer
// Copyright 2026 Google LLC
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
syntax = "proto3";
|
|
|
|
package google.dataflow.v1beta3;
|
|
|
|
import "google/api/annotations.proto";
|
|
import "google/api/client.proto";
|
|
import "google/api/field_behavior.proto";
|
|
import "google/dataflow/v1beta3/environment.proto";
|
|
import "google/dataflow/v1beta3/jobs.proto";
|
|
import "google/rpc/status.proto";
|
|
|
|
option csharp_namespace = "Google.Cloud.Dataflow.V1Beta3";
|
|
option go_package = "cloud.google.com/go/dataflow/apiv1beta3/dataflowpb;dataflowpb";
|
|
option java_multiple_files = true;
|
|
option java_outer_classname = "TemplatesProto";
|
|
option java_package = "com.google.dataflow.v1beta3";
|
|
option php_namespace = "Google\\Cloud\\Dataflow\\V1beta3";
|
|
option ruby_package = "Google::Cloud::Dataflow::V1beta3";
|
|
|
|
// Provides a method to create Cloud Dataflow jobs from templates.
|
|
service TemplatesService {
|
|
option (google.api.default_host) = "dataflow.googleapis.com";
|
|
option (google.api.oauth_scopes) =
|
|
"https://www.googleapis.com/auth/cloud-platform,"
|
|
"https://www.googleapis.com/auth/compute";
|
|
|
|
// Creates a Cloud Dataflow job from a template. Do not enter confidential
|
|
// information when you supply string values using the API.
|
|
//
|
|
// To create a job, we recommend using `projects.locations.templates.create`
|
|
// with a [regional endpoint]
|
|
// (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints). Using
|
|
// `projects.templates.create` is not recommended, because your job will
|
|
// always start in `us-central1`.
|
|
rpc CreateJobFromTemplate(CreateJobFromTemplateRequest) returns (Job) {
|
|
option (google.api.http) = {
|
|
post: "/v1b3/projects/{project_id}/locations/{location}/templates"
|
|
body: "*"
|
|
additional_bindings {
|
|
post: "/v1b3/projects/{project_id}/templates"
|
|
body: "*"
|
|
}
|
|
};
|
|
}
|
|
|
|
// Launches a template.
|
|
//
|
|
// To launch a template, we recommend using
|
|
// `projects.locations.templates.launch` with a [regional endpoint]
|
|
// (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints). Using
|
|
// `projects.templates.launch` is not recommended, because jobs launched
|
|
// from the template will always start in `us-central1`.
|
|
rpc LaunchTemplate(LaunchTemplateRequest) returns (LaunchTemplateResponse) {
|
|
option (google.api.http) = {
|
|
post: "/v1b3/projects/{project_id}/locations/{location}/templates:launch"
|
|
body: "launch_parameters"
|
|
additional_bindings {
|
|
post: "/v1b3/projects/{project_id}/templates:launch"
|
|
body: "launch_parameters"
|
|
}
|
|
};
|
|
}
|
|
|
|
// Get the template associated with a template.
|
|
//
|
|
// To get the template, we recommend using `projects.locations.templates.get`
|
|
// with a [regional endpoint]
|
|
// (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints). Using
|
|
// `projects.templates.get` is not recommended, because only
|
|
// templates that are running in `us-central1` are retrieved.
|
|
rpc GetTemplate(GetTemplateRequest) returns (GetTemplateResponse) {
|
|
option (google.api.http) = {
|
|
get: "/v1b3/projects/{project_id}/locations/{location}/templates:get"
|
|
additional_bindings { get: "/v1b3/projects/{project_id}/templates:get" }
|
|
};
|
|
}
|
|
}
|
|
|
|
// Provides a service for Flex templates.
|
|
service FlexTemplatesService {
|
|
option (google.api.default_host) = "dataflow.googleapis.com";
|
|
option (google.api.oauth_scopes) =
|
|
"https://www.googleapis.com/auth/cloud-platform,"
|
|
"https://www.googleapis.com/auth/compute";
|
|
|
|
// Launch a job with a FlexTemplate.
|
|
rpc LaunchFlexTemplate(LaunchFlexTemplateRequest)
|
|
returns (LaunchFlexTemplateResponse) {
|
|
option (google.api.http) = {
|
|
post: "/v1b3/projects/{project_id}/locations/{location}/flexTemplates:launch"
|
|
body: "*"
|
|
};
|
|
}
|
|
}
|
|
|
|
// Response to the request to launch a job from Flex Template.
|
|
message LaunchFlexTemplateResponse {
|
|
// The job that was launched, if the request was not a dry run and
|
|
// the job was successfully launched.
|
|
Job job = 1;
|
|
}
|
|
|
|
// Container Spec.
|
|
message ContainerSpec {
|
|
// Name of the docker container image. E.g., gcr.io/project/some-image
|
|
string image = 1;
|
|
|
|
// Metadata describing a template including description and validation rules.
|
|
TemplateMetadata metadata = 2;
|
|
|
|
// Required. SDK info of the Flex Template.
|
|
SDKInfo sdk_info = 3;
|
|
|
|
// Default runtime environment for the job.
|
|
FlexTemplateRuntimeEnvironment default_environment = 4;
|
|
|
|
// Secret Manager secret id for username to authenticate to private registry.
|
|
string image_repository_username_secret_id = 5;
|
|
|
|
// Secret Manager secret id for password to authenticate to private registry.
|
|
string image_repository_password_secret_id = 6;
|
|
|
|
// Cloud Storage path to self-signed certificate of private registry.
|
|
string image_repository_cert_path = 7;
|
|
}
|
|
|
|
// Launch FlexTemplate Parameter.
|
|
message LaunchFlexTemplateParameter {
|
|
// Required. The job name to use for the created job. For update job request,
|
|
// job name should be same as the existing running job.
|
|
string job_name = 1;
|
|
|
|
// Launch Mechanism.
|
|
oneof template {
|
|
// Spec about the container image to launch.
|
|
ContainerSpec container_spec = 4;
|
|
|
|
// Cloud Storage path to a file with json serialized ContainerSpec as
|
|
// content.
|
|
string container_spec_gcs_path = 5;
|
|
}
|
|
|
|
// The parameters for FlexTemplate.
|
|
// Ex. {"num_workers":"5"}
|
|
map<string, string> parameters = 2;
|
|
|
|
// Launch options for this flex template job. This is a common set of options
|
|
// across languages and templates. This should not be used to pass job
|
|
// parameters.
|
|
map<string, string> launch_options = 6;
|
|
|
|
// The runtime environment for the FlexTemplate job
|
|
FlexTemplateRuntimeEnvironment environment = 7;
|
|
|
|
// Set this to true if you are sending a request to update a running
|
|
// streaming job. When set, the job name should be the same as the
|
|
// running job.
|
|
bool update = 8;
|
|
|
|
// Use this to pass transform_name_mappings for streaming update jobs.
|
|
// Ex:{"oldTransformName":"newTransformName",...}'
|
|
map<string, string> transform_name_mappings = 9;
|
|
}
|
|
|
|
// The environment values to be set at runtime for flex template.
|
|
message FlexTemplateRuntimeEnvironment {
|
|
// The initial number of Google Compute Engine instances for the job.
|
|
int32 num_workers = 1;
|
|
|
|
// The maximum number of Google Compute Engine instances to be made
|
|
// available to your pipeline during execution, from 1 to 1000.
|
|
int32 max_workers = 2;
|
|
|
|
// The Compute Engine [availability
|
|
// zone](https://cloud.google.com/compute/docs/regions-zones/regions-zones)
|
|
// for launching worker instances to run your pipeline.
|
|
// In the future, worker_zone will take precedence.
|
|
string zone = 3;
|
|
|
|
// The email address of the service account to run the job as.
|
|
string service_account_email = 4;
|
|
|
|
// The Cloud Storage path to use for temporary files.
|
|
// Must be a valid Cloud Storage URL, beginning with `gs://`.
|
|
string temp_location = 5;
|
|
|
|
// The machine type to use for the job. Defaults to the value from the
|
|
// template if not specified.
|
|
string machine_type = 6;
|
|
|
|
// Additional experiment flags for the job.
|
|
repeated string additional_experiments = 7;
|
|
|
|
// Network to which VMs will be assigned. If empty or unspecified,
|
|
// the service will use the network "default".
|
|
string network = 8;
|
|
|
|
// Subnetwork to which VMs will be assigned, if desired. You can specify a
|
|
// subnetwork using either a complete URL or an abbreviated path. Expected to
|
|
// be of the form
|
|
// "https://www.googleapis.com/compute/v1/projects/HOST_PROJECT_ID/regions/REGION/subnetworks/SUBNETWORK"
|
|
// or "regions/REGION/subnetworks/SUBNETWORK". If the subnetwork is located in
|
|
// a Shared VPC network, you must use the complete URL.
|
|
string subnetwork = 9;
|
|
|
|
// Additional user labels to be specified for the job.
|
|
// Keys and values must follow the restrictions specified in the [labeling
|
|
// restrictions](https://cloud.google.com/compute/docs/labeling-resources#restrictions)
|
|
// page.
|
|
// An object containing a list of "key": value pairs.
|
|
// Example: { "name": "wrench", "mass": "1kg", "count": "3" }.
|
|
map<string, string> additional_user_labels = 10;
|
|
|
|
// Name for the Cloud KMS key for the job.
|
|
// Key format is:
|
|
// projects/<project>/locations/<location>/keyRings/<keyring>/cryptoKeys/<key>
|
|
string kms_key_name = 11;
|
|
|
|
// Configuration for VM IPs.
|
|
WorkerIPAddressConfiguration ip_configuration = 12;
|
|
|
|
// The Compute Engine region
|
|
// (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in
|
|
// which worker processing should occur, e.g. "us-west1". Mutually exclusive
|
|
// with worker_zone. If neither worker_region nor worker_zone is specified,
|
|
// default to the control plane's region.
|
|
string worker_region = 13;
|
|
|
|
// The Compute Engine zone
|
|
// (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in
|
|
// which worker processing should occur, e.g. "us-west1-a". Mutually exclusive
|
|
// with worker_region. If neither worker_region nor worker_zone is specified,
|
|
// a zone in the control plane's region is chosen based on available capacity.
|
|
// If both `worker_zone` and `zone` are set, `worker_zone` takes precedence.
|
|
string worker_zone = 14;
|
|
|
|
// Whether to enable Streaming Engine for the job.
|
|
bool enable_streaming_engine = 15;
|
|
|
|
// Set FlexRS goal for the job.
|
|
// https://cloud.google.com/dataflow/docs/guides/flexrs
|
|
FlexResourceSchedulingGoal flexrs_goal = 16;
|
|
|
|
// The Cloud Storage path for staging local files.
|
|
// Must be a valid Cloud Storage URL, beginning with `gs://`.
|
|
string staging_location = 17;
|
|
|
|
// Docker registry location of container image to use for the 'worker harness.
|
|
// Default is the container for the version of the SDK. Note this field is
|
|
// only valid for portable pipelines.
|
|
string sdk_container_image = 18;
|
|
|
|
// Worker disk size, in gigabytes.
|
|
int32 disk_size_gb = 20;
|
|
|
|
// The algorithm to use for autoscaling
|
|
AutoscalingAlgorithm autoscaling_algorithm = 21;
|
|
|
|
// If true, when processing time is spent almost entirely
|
|
// on garbage collection (GC), saves a heap dump before ending the thread
|
|
// or process. If false, ends the thread or process without saving a heap
|
|
// dump. Does not save a heap dump when the Java Virtual Machine (JVM) has an
|
|
// out of memory error during processing. The location of the heap file is
|
|
// either echoed back to the user, or the user is given the opportunity to
|
|
// download the heap file.
|
|
bool dump_heap_on_oom = 22;
|
|
|
|
// Cloud Storage bucket (directory) to upload heap dumps to.
|
|
// Enabling this field implies that `dump_heap_on_oom` is set to true.
|
|
string save_heap_dumps_to_gcs_path = 23;
|
|
|
|
// The machine type to use for launching the job. The default is
|
|
// n1-standard-1.
|
|
string launcher_machine_type = 24;
|
|
|
|
// If true serial port logging will be enabled for the launcher VM.
|
|
bool enable_launcher_vm_serial_port_logging = 25;
|
|
|
|
// Optional. Specifies the Streaming Engine message processing guarantees.
|
|
// Reduces cost and latency but might result in duplicate messages committed
|
|
// to storage. Designed to run simple mapping streaming ETL jobs at the lowest
|
|
// cost. For example, Change Data Capture (CDC) to BigQuery is a canonical use
|
|
// case. For more information, see
|
|
// [Set the pipeline streaming
|
|
// mode](https://cloud.google.com/dataflow/docs/guides/streaming-modes).
|
|
optional StreamingMode streaming_mode = 26
|
|
[(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Additional pipeline option flags for the job.
|
|
repeated string additional_pipeline_options = 27
|
|
[(google.api.field_behavior) = OPTIONAL];
|
|
}
|
|
|
|
// A request to launch a Cloud Dataflow job from a FlexTemplate.
|
|
message LaunchFlexTemplateRequest {
|
|
// Required. The ID of the Cloud Platform project that the job belongs to.
|
|
string project_id = 1;
|
|
|
|
// Required. Parameter to launch a job form Flex Template.
|
|
LaunchFlexTemplateParameter launch_parameter = 2;
|
|
|
|
// Required. The [regional endpoint]
|
|
// (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) to
|
|
// which to direct the request. E.g., us-central1, us-west1.
|
|
string location = 3;
|
|
|
|
// If true, the request is validated but not actually executed.
|
|
// Defaults to false.
|
|
bool validate_only = 4;
|
|
}
|
|
|
|
// The environment values to set at runtime.
|
|
message RuntimeEnvironment {
|
|
// Optional. The initial number of Google Compute Engine instances for the
|
|
// job. The default value is 11.
|
|
int32 num_workers = 11;
|
|
|
|
// Optional. The maximum number of Google Compute Engine instances to be made
|
|
// available to your pipeline during execution, from 1 to 1000. The default
|
|
// value is 1.
|
|
int32 max_workers = 1;
|
|
|
|
// Optional. The Compute Engine [availability
|
|
// zone](https://cloud.google.com/compute/docs/regions-zones/regions-zones)
|
|
// for launching worker instances to run your pipeline.
|
|
// In the future, worker_zone will take precedence.
|
|
string zone = 2;
|
|
|
|
// Optional. The email address of the service account to run the job as.
|
|
string service_account_email = 3;
|
|
|
|
// Required. The Cloud Storage path to use for temporary files.
|
|
// Must be a valid Cloud Storage URL, beginning with `gs://`.
|
|
string temp_location = 4;
|
|
|
|
// Optional. Whether to bypass the safety checks for the job's temporary
|
|
// directory. Use with caution.
|
|
bool bypass_temp_dir_validation = 5;
|
|
|
|
// Optional. The machine type to use for the job. Defaults to the value from
|
|
// the template if not specified.
|
|
string machine_type = 6;
|
|
|
|
// Optional. Additional experiment flags for the job, specified with the
|
|
// `--experiments` option.
|
|
repeated string additional_experiments = 7;
|
|
|
|
// Optional. Network to which VMs will be assigned. If empty or unspecified,
|
|
// the service will use the network "default".
|
|
string network = 8;
|
|
|
|
// Optional. Subnetwork to which VMs will be assigned, if desired. You can
|
|
// specify a subnetwork using either a complete URL or an abbreviated path.
|
|
// Expected to be of the form
|
|
// "https://www.googleapis.com/compute/v1/projects/HOST_PROJECT_ID/regions/REGION/subnetworks/SUBNETWORK"
|
|
// or "regions/REGION/subnetworks/SUBNETWORK". If the subnetwork is located in
|
|
// a Shared VPC network, you must use the complete URL.
|
|
string subnetwork = 9;
|
|
|
|
// Optional. Additional user labels to be specified for the job.
|
|
// Keys and values should follow the restrictions specified in the [labeling
|
|
// restrictions](https://cloud.google.com/compute/docs/labeling-resources#restrictions)
|
|
// page.
|
|
// An object containing a list of "key": value pairs.
|
|
// Example: { "name": "wrench", "mass": "1kg", "count": "3" }.
|
|
map<string, string> additional_user_labels = 10;
|
|
|
|
// Optional. Name for the Cloud KMS key for the job.
|
|
// Key format is:
|
|
// projects/<project>/locations/<location>/keyRings/<keyring>/cryptoKeys/<key>
|
|
string kms_key_name = 12;
|
|
|
|
// Optional. Configuration for VM IPs.
|
|
WorkerIPAddressConfiguration ip_configuration = 14;
|
|
|
|
// Required. The Compute Engine region
|
|
// (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in
|
|
// which worker processing should occur, e.g. "us-west1". Mutually exclusive
|
|
// with worker_zone. If neither worker_region nor worker_zone is specified,
|
|
// default to the control plane's region.
|
|
string worker_region = 15;
|
|
|
|
// Optional. The Compute Engine zone
|
|
// (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in
|
|
// which worker processing should occur, e.g. "us-west1-a". Mutually exclusive
|
|
// with worker_region. If neither worker_region nor worker_zone is specified,
|
|
// a zone in the control plane's region is chosen based on available capacity.
|
|
// If both `worker_zone` and `zone` are set, `worker_zone` takes precedence.
|
|
string worker_zone = 16;
|
|
|
|
// Optional. Whether to enable Streaming Engine for the job.
|
|
bool enable_streaming_engine = 17;
|
|
|
|
// Optional. The disk size, in gigabytes, to use on each remote Compute Engine
|
|
// worker instance.
|
|
int32 disk_size_gb = 18;
|
|
|
|
// Optional. Specifies the Streaming Engine message processing guarantees.
|
|
// Reduces cost and latency but might result in duplicate messages committed
|
|
// to storage. Designed to run simple mapping streaming ETL jobs at the lowest
|
|
// cost. For example, Change Data Capture (CDC) to BigQuery is a canonical use
|
|
// case. For more information, see
|
|
// [Set the pipeline streaming
|
|
// mode](https://cloud.google.com/dataflow/docs/guides/streaming-modes).
|
|
optional StreamingMode streaming_mode = 19
|
|
[(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Additional pipeline option flags for the job.
|
|
repeated string additional_pipeline_options = 20
|
|
[(google.api.field_behavior) = OPTIONAL];
|
|
}
|
|
|
|
// ParameterMetadataEnumOption specifies the option shown in the enum form.
|
|
message ParameterMetadataEnumOption {
|
|
// Required. The value of the enum option.
|
|
string value = 1 [(google.api.field_behavior) = REQUIRED];
|
|
|
|
// Optional. The label to display for the enum option.
|
|
string label = 2 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. The description to display for the enum option.
|
|
string description = 3 [(google.api.field_behavior) = OPTIONAL];
|
|
}
|
|
|
|
// Metadata for a specific parameter.
|
|
message ParameterMetadata {
|
|
// Required. The name of the parameter.
|
|
string name = 1;
|
|
|
|
// Required. The label to display for the parameter.
|
|
string label = 2;
|
|
|
|
// Required. The help text to display for the parameter.
|
|
string help_text = 3;
|
|
|
|
// Optional. Whether the parameter is optional. Defaults to false.
|
|
bool is_optional = 4;
|
|
|
|
// Optional. Regexes that the parameter must match.
|
|
repeated string regexes = 5;
|
|
|
|
// Optional. The type of the parameter.
|
|
// Used for selecting input picker.
|
|
ParameterType param_type = 6;
|
|
|
|
// Optional. Additional metadata for describing this parameter.
|
|
map<string, string> custom_metadata = 7;
|
|
|
|
// Optional. Specifies a group name for this parameter to be rendered under.
|
|
// Group header text will be rendered exactly as specified in this field. Only
|
|
// considered when parent_name is NOT provided.
|
|
string group_name = 8;
|
|
|
|
// Optional. Specifies the name of the parent parameter. Used in conjunction
|
|
// with 'parent_trigger_values' to make this parameter conditional (will only
|
|
// be rendered conditionally). Should be mappable to a ParameterMetadata.name
|
|
// field.
|
|
string parent_name = 9;
|
|
|
|
// Optional. The value(s) of the 'parent_name' parameter which will trigger
|
|
// this parameter to be shown. If left empty, ANY non-empty value in
|
|
// parent_name will trigger this parameter to be shown. Only considered when
|
|
// this parameter is conditional (when 'parent_name' has been provided).
|
|
repeated string parent_trigger_values = 10;
|
|
|
|
// Optional. The options shown when ENUM ParameterType is specified.
|
|
repeated ParameterMetadataEnumOption enum_options = 11;
|
|
|
|
// Optional. The default values will pre-populate the parameter with the
|
|
// given value from the proto. If default_value is left empty, the parameter
|
|
// will be populated with a default of the relevant type, e.g. false for a
|
|
// boolean.
|
|
string default_value = 12;
|
|
|
|
// Optional. Whether the parameter should be hidden in the UI.
|
|
bool hidden_ui = 13;
|
|
}
|
|
|
|
// Metadata describing a template.
|
|
message TemplateMetadata {
|
|
// Required. The name of the template.
|
|
string name = 1;
|
|
|
|
// Optional. A description of the template.
|
|
string description = 2;
|
|
|
|
// The parameters for the template.
|
|
repeated ParameterMetadata parameters = 3;
|
|
|
|
// Optional. Indicates if the template is streaming or not.
|
|
bool streaming = 5;
|
|
|
|
// Optional. Indicates if the streaming template supports at least once mode.
|
|
bool supports_at_least_once = 6;
|
|
|
|
// Optional. Indicates if the streaming template supports exactly once mode.
|
|
bool supports_exactly_once = 7;
|
|
|
|
// Optional. Indicates the default streaming mode for a streaming template.
|
|
// Only valid if both supports_at_least_once and supports_exactly_once are
|
|
// true. Possible values: UNSPECIFIED, EXACTLY_ONCE and AT_LEAST_ONCE
|
|
string default_streaming_mode = 8;
|
|
}
|
|
|
|
// SDK Information.
|
|
message SDKInfo {
|
|
// SDK Language.
|
|
enum Language {
|
|
// UNKNOWN Language.
|
|
UNKNOWN = 0;
|
|
|
|
// Java.
|
|
JAVA = 1;
|
|
|
|
// Python.
|
|
PYTHON = 2;
|
|
|
|
// Go.
|
|
GO = 3;
|
|
|
|
// YAML.
|
|
YAML = 4;
|
|
}
|
|
|
|
// Required. The SDK Language.
|
|
Language language = 1;
|
|
|
|
// Optional. The SDK version.
|
|
string version = 2;
|
|
}
|
|
|
|
// RuntimeMetadata describing a runtime environment.
|
|
message RuntimeMetadata {
|
|
// SDK Info for the template.
|
|
SDKInfo sdk_info = 1;
|
|
|
|
// The parameters for the template.
|
|
repeated ParameterMetadata parameters = 2;
|
|
}
|
|
|
|
// A request to create a Cloud Dataflow job from a template.
|
|
message CreateJobFromTemplateRequest {
|
|
// Required. The ID of the Cloud Platform project that the job belongs to.
|
|
string project_id = 1;
|
|
|
|
// Required. The job name to use for the created job.
|
|
string job_name = 4;
|
|
|
|
// The template from which to create the job.
|
|
oneof template {
|
|
// Required. A Cloud Storage path to the template from which to
|
|
// create the job.
|
|
// Must be a valid Cloud Storage URL, beginning with `gs://`.
|
|
string gcs_path = 2;
|
|
}
|
|
|
|
// The runtime parameters to pass to the job.
|
|
map<string, string> parameters = 3;
|
|
|
|
// The runtime environment for the job.
|
|
RuntimeEnvironment environment = 5;
|
|
|
|
// The [regional endpoint]
|
|
// (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) to
|
|
// which to direct the request.
|
|
string location = 6;
|
|
}
|
|
|
|
// A request to retrieve a Cloud Dataflow job template.
|
|
message GetTemplateRequest {
|
|
// The various views of a template that may be retrieved.
|
|
enum TemplateView {
|
|
// Template view that retrieves only the metadata associated with the
|
|
// template.
|
|
METADATA_ONLY = 0;
|
|
}
|
|
|
|
// Required. The ID of the Cloud Platform project that the job belongs to.
|
|
string project_id = 1;
|
|
|
|
// The template from which to create the job.
|
|
oneof template {
|
|
// Required. A Cloud Storage path to the template from which to
|
|
// create the job.
|
|
// Must be valid Cloud Storage URL, beginning with 'gs://'.
|
|
string gcs_path = 2;
|
|
}
|
|
|
|
// The view to retrieve. Defaults to METADATA_ONLY.
|
|
TemplateView view = 3;
|
|
|
|
// The [regional endpoint]
|
|
// (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) to
|
|
// which to direct the request.
|
|
string location = 4;
|
|
}
|
|
|
|
// The response to a GetTemplate request.
|
|
message GetTemplateResponse {
|
|
// Template Type.
|
|
enum TemplateType {
|
|
// Unknown Template Type.
|
|
UNKNOWN = 0;
|
|
|
|
// Legacy Template.
|
|
LEGACY = 1;
|
|
|
|
// Flex Template.
|
|
FLEX = 2;
|
|
}
|
|
|
|
// The status of the get template request. Any problems with the
|
|
// request will be indicated in the error_details.
|
|
google.rpc.Status status = 1;
|
|
|
|
// The template metadata describing the template name, available
|
|
// parameters, etc.
|
|
TemplateMetadata metadata = 2;
|
|
|
|
// Template Type.
|
|
TemplateType template_type = 3;
|
|
|
|
// Describes the runtime metadata with SDKInfo and available parameters.
|
|
RuntimeMetadata runtime_metadata = 4;
|
|
}
|
|
|
|
// Parameters to provide to the template being launched. Note that the
|
|
// [metadata in the pipeline code]
|
|
// (https://cloud.google.com/dataflow/docs/guides/templates/creating-templates#metadata)
|
|
// determines which runtime parameters are valid.
|
|
message LaunchTemplateParameters {
|
|
// Required. The job name to use for the created job.
|
|
//
|
|
// The name must match the regular expression
|
|
// `[a-z]([-a-z0-9]{0,1022}[a-z0-9])?`
|
|
string job_name = 1;
|
|
|
|
// The runtime parameters to pass to the job.
|
|
map<string, string> parameters = 2;
|
|
|
|
// The runtime environment for the job.
|
|
RuntimeEnvironment environment = 3;
|
|
|
|
// If set, replace the existing pipeline with the name specified by jobName
|
|
// with this pipeline, preserving state.
|
|
bool update = 4;
|
|
|
|
// Only applicable when updating a pipeline. Map of transform name prefixes of
|
|
// the job to be replaced to the corresponding name prefixes of the new job.
|
|
map<string, string> transform_name_mapping = 5;
|
|
}
|
|
|
|
// A request to launch a template.
|
|
message LaunchTemplateRequest {
|
|
// Required. The ID of the Cloud Platform project that the job belongs to.
|
|
string project_id = 1;
|
|
|
|
// If true, the request is validated but not actually executed.
|
|
// Defaults to false.
|
|
bool validate_only = 2;
|
|
|
|
// The template to use to create the job.
|
|
oneof template {
|
|
// A Cloud Storage path to the template to use to create
|
|
// the job.
|
|
// Must be valid Cloud Storage URL, beginning with `gs://`.
|
|
string gcs_path = 3;
|
|
|
|
// Parameters for launching a dynamic template.
|
|
DynamicTemplateLaunchParams dynamic_template = 6;
|
|
}
|
|
|
|
// The parameters of the template to launch. Part of the
|
|
// body of the POST request.
|
|
LaunchTemplateParameters launch_parameters = 4;
|
|
|
|
// The [regional endpoint]
|
|
// (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) to
|
|
// which to direct the request.
|
|
string location = 5;
|
|
}
|
|
|
|
// Response to the request to launch a template.
|
|
message LaunchTemplateResponse {
|
|
// The job that was launched, if the request was not a dry run and
|
|
// the job was successfully launched.
|
|
Job job = 1;
|
|
}
|
|
|
|
// Used in the error_details field of a google.rpc.Status message, this
|
|
// indicates problems with the template parameter.
|
|
message InvalidTemplateParameters {
|
|
// A specific template-parameter violation.
|
|
message ParameterViolation {
|
|
// The parameter that failed to validate.
|
|
string parameter = 1;
|
|
|
|
// A description of why the parameter failed to validate.
|
|
string description = 2;
|
|
}
|
|
|
|
// Describes all parameter violations in a template request.
|
|
repeated ParameterViolation parameter_violations = 1;
|
|
}
|
|
|
|
// Parameters to pass when launching a dynamic template.
|
|
message DynamicTemplateLaunchParams {
|
|
// Path to the dynamic template specification file on Cloud Storage.
|
|
// The file must be a JSON serialized `DynamicTemplateFileSpec` object.
|
|
string gcs_path = 1;
|
|
|
|
// Cloud Storage path for staging dependencies.
|
|
// Must be a valid Cloud Storage URL, beginning with `gs://`.
|
|
string staging_location = 2;
|
|
}
|
|
|
|
// ParameterType specifies what kind of input we need for this parameter.
|
|
enum ParameterType {
|
|
// Default input type.
|
|
DEFAULT = 0;
|
|
|
|
// The parameter specifies generic text input.
|
|
TEXT = 1;
|
|
|
|
// The parameter specifies a Cloud Storage Bucket to read from.
|
|
GCS_READ_BUCKET = 2;
|
|
|
|
// The parameter specifies a Cloud Storage Bucket to write to.
|
|
GCS_WRITE_BUCKET = 3;
|
|
|
|
// The parameter specifies a Cloud Storage file path to read from.
|
|
GCS_READ_FILE = 4;
|
|
|
|
// The parameter specifies a Cloud Storage file path to write to.
|
|
GCS_WRITE_FILE = 5;
|
|
|
|
// The parameter specifies a Cloud Storage folder path to read from.
|
|
GCS_READ_FOLDER = 6;
|
|
|
|
// The parameter specifies a Cloud Storage folder to write to.
|
|
GCS_WRITE_FOLDER = 7;
|
|
|
|
// The parameter specifies a Pub/Sub Topic.
|
|
PUBSUB_TOPIC = 8;
|
|
|
|
// The parameter specifies a Pub/Sub Subscription.
|
|
PUBSUB_SUBSCRIPTION = 9;
|
|
|
|
// The parameter specifies a BigQuery table.
|
|
BIGQUERY_TABLE = 10;
|
|
|
|
// The parameter specifies a JavaScript UDF in Cloud Storage.
|
|
JAVASCRIPT_UDF_FILE = 11;
|
|
|
|
// The parameter specifies a Service Account email.
|
|
SERVICE_ACCOUNT = 12;
|
|
|
|
// The parameter specifies a Machine Type.
|
|
MACHINE_TYPE = 13;
|
|
|
|
// The parameter specifies a KMS Key name.
|
|
KMS_KEY_NAME = 14;
|
|
|
|
// The parameter specifies a Worker Region.
|
|
WORKER_REGION = 15;
|
|
|
|
// The parameter specifies a Worker Zone.
|
|
WORKER_ZONE = 16;
|
|
|
|
// The parameter specifies a boolean input.
|
|
BOOLEAN = 17;
|
|
|
|
// The parameter specifies an enum input.
|
|
ENUM = 18;
|
|
|
|
// The parameter specifies a number input.
|
|
NUMBER = 19;
|
|
|
|
// Deprecated. Please use KAFKA_READ_TOPIC instead.
|
|
KAFKA_TOPIC = 20 [deprecated = true];
|
|
|
|
// The parameter specifies the fully-qualified name of an Apache Kafka topic.
|
|
// This can be either a Google Managed Kafka topic or a non-managed Kafka
|
|
// topic.
|
|
KAFKA_READ_TOPIC = 21;
|
|
|
|
// The parameter specifies the fully-qualified name of an Apache Kafka topic.
|
|
// This can be an existing Google Managed Kafka topic, the name for a new
|
|
// Google Managed Kafka topic, or an existing non-managed Kafka topic.
|
|
KAFKA_WRITE_TOPIC = 22;
|
|
}
|