From b4da4fd5eee314df550791fc287c1f1950127ee4 Mon Sep 17 00:00:00 2001 From: Google APIs Date: Fri, 17 Sep 2021 10:32:22 -0700 Subject: [PATCH] feat: add BigQuery Storage Write API v1 Committer: @yirutang PiperOrigin-RevId: 397350004 --- google/cloud/bigquery/storage/v1/BUILD.bazel | 95 ++--- .../bigquerystorage_grpc_service_config.json | 39 +- .../storage/v1/bigquerystorage_v1.yaml | 20 +- .../cloud/bigquery/storage/v1/protobuf.proto | 48 +++ .../cloud/bigquery/storage/v1/storage.proto | 361 +++++++++++++++++- google/cloud/bigquery/storage/v1/stream.proto | 48 +++ google/cloud/bigquery/storage/v1/table.proto | 164 ++++++++ 7 files changed, 726 insertions(+), 49 deletions(-) create mode 100644 google/cloud/bigquery/storage/v1/protobuf.proto create mode 100644 google/cloud/bigquery/storage/v1/table.proto diff --git a/google/cloud/bigquery/storage/v1/BUILD.bazel b/google/cloud/bigquery/storage/v1/BUILD.bazel index bc8f21183b..15674bd20f 100644 --- a/google/cloud/bigquery/storage/v1/BUILD.bazel +++ b/google/cloud/bigquery/storage/v1/BUILD.bazel @@ -12,15 +12,21 @@ proto_library( srcs = [ "arrow.proto", "avro.proto", + "protobuf.proto", "storage.proto", "stream.proto", + "table.proto", ], deps = [ "//google/api:annotations_proto", "//google/api:client_proto", "//google/api:field_behavior_proto", "//google/api:resource_proto", + "//google/rpc:status_proto", + "@com_google_protobuf//:descriptor_proto", + "@com_google_protobuf//:empty_proto", "@com_google_protobuf//:timestamp_proto", + "@com_google_protobuf//:wrappers_proto", ], ) @@ -105,6 +111,7 @@ go_proto_library( protos = [":storage_proto"], deps = [ "//google/api:annotations_go_proto", + "//google/rpc:status_go_proto", ], ) @@ -236,53 +243,55 @@ nodejs_gapic_assembly_pkg( ############################################################################## # Ruby ############################################################################## -load( - "@com_google_googleapis_imports//:imports.bzl", - "ruby_cloud_gapic_library", - "ruby_gapic_assembly_pkg", - "ruby_grpc_library", - "ruby_proto_library", -) +# disabled due to https://github.com/protocolbuffers/protobuf/issues/7992 +# +# load( +# "@com_google_googleapis_imports//:imports.bzl", +# "ruby_cloud_gapic_library", +# "ruby_gapic_assembly_pkg", +# "ruby_grpc_library", +# "ruby_proto_library", +# ) -ruby_proto_library( - name = "storage_ruby_proto", - deps = [":storage_proto"], -) +# ruby_proto_library( +# name = "storage_ruby_proto", +# deps = [":storage_proto"], +# ) -ruby_grpc_library( - name = "storage_ruby_grpc", - srcs = [":storage_proto"], - deps = [":storage_ruby_proto"], -) +# ruby_grpc_library( +# name = "storage_ruby_grpc", +# srcs = [":storage_proto"], +# deps = [":storage_ruby_proto"], +# ) -ruby_cloud_gapic_library( - name = "storage_ruby_gapic", - srcs = [":storage_proto_with_info"], - extra_protoc_parameters = [ - "ruby-cloud-gem-name=google-cloud-bigquery-storage-v1", - "ruby-cloud-env-prefix=BIGQUERY_STORAGE", - "ruby-cloud-product-url=https://cloud.google.com/bigquery/docs/reference/storage", - "ruby-cloud-api-id=bigquerystorage.googleapis.com", - "ruby-cloud-api-shortname=bigquerystorage", - ], - grpc_service_config = "bigquerystorage_grpc_service_config.json", - ruby_cloud_description = "The BigQuery Storage API provides fast access to BigQuery managed storage.", - ruby_cloud_title = "BigQuery Storage V1", - deps = [ - ":storage_ruby_grpc", - ":storage_ruby_proto", - ], -) +# ruby_cloud_gapic_library( +# name = "storage_ruby_gapic", +# srcs = [":storage_proto_with_info"], +# extra_protoc_parameters = [ +# "ruby-cloud-gem-name=google-cloud-bigquery-storage-v1", +# "ruby-cloud-env-prefix=BIGQUERY_STORAGE", +# "ruby-cloud-product-url=https://cloud.google.com/bigquery/docs/reference/storage", +# "ruby-cloud-api-id=bigquerystorage.googleapis.com", +# "ruby-cloud-api-shortname=bigquerystorage", +# ], +# grpc_service_config = "bigquerystorage_grpc_service_config.json", +# ruby_cloud_description = "The BigQuery Storage API provides fast access to BigQuery managed storage.", +# ruby_cloud_title = "BigQuery Storage V1", +# deps = [ +# ":storage_ruby_grpc", +# ":storage_ruby_proto", +# ], +# ) -# Open Source Packages -ruby_gapic_assembly_pkg( - name = "google-cloud-bigquery-storage-v1-ruby", - deps = [ - ":storage_ruby_gapic", - ":storage_ruby_grpc", - ":storage_ruby_proto", - ], -) +# # Open Source Packages +# ruby_gapic_assembly_pkg( +# name = "google-cloud-bigquery-storage-v1-ruby", +# deps = [ +# ":storage_ruby_gapic", +# ":storage_ruby_grpc", +# ":storage_ruby_proto", +# ], +# ) ############################################################################## # C# diff --git a/google/cloud/bigquery/storage/v1/bigquerystorage_grpc_service_config.json b/google/cloud/bigquery/storage/v1/bigquerystorage_grpc_service_config.json index 2efc373b12..300e9e26d5 100644 --- a/google/cloud/bigquery/storage/v1/bigquerystorage_grpc_service_config.json +++ b/google/cloud/bigquery/storage/v1/bigquerystorage_grpc_service_config.json @@ -34,12 +34,27 @@ "UNAVAILABLE" ] } - }, - { + }, { "name": [ { - "service": "google.cloud.bigquery.storage.v1.BigQueryRead", - "method": "SplitReadStream" + "service": "google.cloud.bigquery.storage.v1.BigQueryWrite", + "method": "CreateWriteStream" + }, + { + "service": "google.cloud.bigquery.storage.v1.BigQueryWrite", + "method": "BatchCommitWriteStreams" + }, + { + "service": "google.cloud.bigquery.storage.v1.BigQueryWrite", + "method": "FinalizeWriteStream" + }, + { + "service": "google.cloud.bigquery.storage.v1.BigQueryWrite", + "method": "GetWriteStream" + }, + { + "service": "google.cloud.bigquery.storage.v1.BigQueryWrite", + "method": "FlushRows" } ], "timeout": "600s", @@ -52,6 +67,22 @@ "UNAVAILABLE" ] } + }, { + "name": [ + { + "service": "google.cloud.bigquery.storage.v1.BigQueryWrite", + "method": "AppendRows" + } + ], + "timeout": "86400s", + "retryPolicy": { + "initialBackoff": "0.100s", + "maxBackoff": "60s", + "backoffMultiplier": 1.3, + "retryableStatusCodes": [ + "UNAVAILABLE" + ] + } } ] } diff --git a/google/cloud/bigquery/storage/v1/bigquerystorage_v1.yaml b/google/cloud/bigquery/storage/v1/bigquerystorage_v1.yaml index f671c4a51d..1c35a13dcc 100644 --- a/google/cloud/bigquery/storage/v1/bigquerystorage_v1.yaml +++ b/google/cloud/bigquery/storage/v1/bigquerystorage_v1.yaml @@ -5,6 +5,7 @@ title: BigQuery Storage API apis: - name: google.cloud.bigquery.storage.v1.BigQueryRead +- name: google.cloud.bigquery.storage.v1.BigQueryWrite backend: rules: @@ -14,6 +15,18 @@ backend: deadline: 21600.0 - selector: google.cloud.bigquery.storage.v1.BigQueryRead.SplitReadStream deadline: 120.0 + - selector: google.cloud.bigquery.storage.v1.BigQueryWrite.CreateWriteStram + deadline: 120.0 + - selector: google.cloud.bigquery.storage.v1.BigQueryWrite.FinalizeWriteStream + deadline: 120.0 + - selector: google.cloud.bigquery.storage.v1.BigQueryWrite.BatchCommitWriteStreams + deadline: 120.0 + - selector: google.cloud.bigquery.storage.v1.BigQueryWrite.GetWriteStream + deadline: 120.0 + - selector: google.cloud.bigquery.storage.v1.BigQueryWrite.FlushRows + deadline: 120.0 + - selector: google.cloud.bigquery.storage.v1.BigQueryWrite.AppendRows + deadline: 86400.0 authentication: rules: @@ -21,5 +34,10 @@ authentication: oauth: canonical_scopes: |- https://www.googleapis.com/auth/bigquery, - https://www.googleapis.com/auth/bigquery.readonly, + https://www.googleapis.com/auth/cloud-platform + - selector: 'google.cloud.bigquery.storage.v1.BigQueryWrite.*' + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/bigquery, + https://www.googleapis.com/auth/bigquery.insertdata, https://www.googleapis.com/auth/cloud-platform diff --git a/google/cloud/bigquery/storage/v1/protobuf.proto b/google/cloud/bigquery/storage/v1/protobuf.proto new file mode 100644 index 0000000000..f987467dd9 --- /dev/null +++ b/google/cloud/bigquery/storage/v1/protobuf.proto @@ -0,0 +1,48 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.cloud.bigquery.storage.v1; + +import "google/protobuf/descriptor.proto"; + +option csharp_namespace = "Google.Cloud.BigQuery.Storage.V1"; +option go_package = "google.golang.org/genproto/googleapis/cloud/bigquery/storage/v1;storage"; +option java_multiple_files = true; +option java_outer_classname = "ProtoBufProto"; +option java_package = "com.google.cloud.bigquery.storage.v1"; +option php_namespace = "Google\\Cloud\\BigQuery\\Storage\\V1"; + +// ProtoSchema describes the schema of the serialized protocol buffer data rows. +message ProtoSchema { + // Descriptor for input message. The provided descriptor must be self + // contained, such that data rows sent can be fully decoded using only the + // single descriptor. For data rows that are compositions of multiple + // independent messages, this means the descriptor may need to be transformed + // to only use nested types: + // https://developers.google.com/protocol-buffers/docs/proto#nested + // + // For additional information for how proto types and values map onto BigQuery + // see: https://cloud.google.com/bigquery/docs/write-api#data_type_conversions + google.protobuf.DescriptorProto proto_descriptor = 1; +} + +message ProtoRows { + // A sequence of rows serialized as a Protocol Buffer. + // + // See https://developers.google.com/protocol-buffers/docs/overview for more + // information on deserializing this field. + repeated bytes serialized_rows = 1; +} diff --git a/google/cloud/bigquery/storage/v1/storage.proto b/google/cloud/bigquery/storage/v1/storage.proto index e86ad253cf..ab5a46cf18 100644 --- a/google/cloud/bigquery/storage/v1/storage.proto +++ b/google/cloud/bigquery/storage/v1/storage.proto @@ -22,7 +22,12 @@ import "google/api/field_behavior.proto"; import "google/api/resource.proto"; import "google/cloud/bigquery/storage/v1/arrow.proto"; import "google/cloud/bigquery/storage/v1/avro.proto"; +import "google/cloud/bigquery/storage/v1/protobuf.proto"; import "google/cloud/bigquery/storage/v1/stream.proto"; +import "google/cloud/bigquery/storage/v1/table.proto"; +import "google/protobuf/timestamp.proto"; +import "google/protobuf/wrappers.proto"; +import "google/rpc/status.proto"; option csharp_namespace = "Google.Cloud.BigQuery.Storage.V1"; option go_package = "google.golang.org/genproto/googleapis/cloud/bigquery/storage/v1;storage"; @@ -105,6 +110,121 @@ service BigQueryRead { } } +// BigQuery Write API. +// +// The Write API can be used to write data to BigQuery. +// +// For supplementary information about the Write API, see: +// https://cloud.google.com/bigquery/docs/write-api +service BigQueryWrite { + option (google.api.default_host) = "bigquerystorage.googleapis.com"; + option (google.api.oauth_scopes) = + "https://www.googleapis.com/auth/bigquery," + "https://www.googleapis.com/auth/bigquery.insertdata," + "https://www.googleapis.com/auth/cloud-platform"; + + // Creates a write stream to the given table. + // Additionally, every table has a special stream named '_default' + // to which data can be written. This stream doesn't need to be created using + // CreateWriteStream. It is a stream that can be used simultaneously by any + // number of clients. Data written to this stream is considered committed as + // soon as an acknowledgement is received. + rpc CreateWriteStream(CreateWriteStreamRequest) returns (WriteStream) { + option (google.api.http) = { + post: "/v1/{parent=projects/*/datasets/*/tables/*}" + body: "write_stream" + }; + option (google.api.method_signature) = "parent,write_stream"; + } + + // Appends data to the given stream. + // + // If `offset` is specified, the `offset` is checked against the end of + // stream. The server returns `OUT_OF_RANGE` in `AppendRowsResponse` if an + // attempt is made to append to an offset beyond the current end of the stream + // or `ALREADY_EXISTS` if user provides an `offset` that has already been + // written to. User can retry with adjusted offset within the same RPC + // connection. If `offset` is not specified, append happens at the end of the + // stream. + // + // The response contains an optional offset at which the append + // happened. No offset information will be returned for appends to a + // default stream. + // + // Responses are received in the same order in which requests are sent. + // There will be one response for each successful inserted request. Responses + // may optionally embed error information if the originating AppendRequest was + // not successfully processed. + // + // The specifics of when successfully appended data is made visible to the + // table are governed by the type of stream: + // + // * For COMMITTED streams (which includes the default stream), data is + // visible immediately upon successful append. + // + // * For BUFFERED streams, data is made visible via a subsequent `FlushRows` + // rpc which advances a cursor to a newer offset in the stream. + // + // * For PENDING streams, data is not made visible until the stream itself is + // finalized (via the `FinalizeWriteStream` rpc), and the stream is explicitly + // committed via the `BatchCommitWriteStreams` rpc. + rpc AppendRows(stream AppendRowsRequest) returns (stream AppendRowsResponse) { + option (google.api.http) = { + post: "/v1/{write_stream=projects/*/datasets/*/tables/*/streams/*}" + body: "*" + }; + option (google.api.method_signature) = "write_stream"; + } + + // Gets information about a write stream. + rpc GetWriteStream(GetWriteStreamRequest) returns (WriteStream) { + option (google.api.http) = { + post: "/v1/{name=projects/*/datasets/*/tables/*/streams/*}" + body: "*" + }; + option (google.api.method_signature) = "name"; + } + + // Finalize a write stream so that no new data can be appended to the + // stream. Finalize is not supported on the '_default' stream. + rpc FinalizeWriteStream(FinalizeWriteStreamRequest) returns (FinalizeWriteStreamResponse) { + option (google.api.http) = { + post: "/v1/{name=projects/*/datasets/*/tables/*/streams/*}" + body: "*" + }; + option (google.api.method_signature) = "name"; + } + + // Atomically commits a group of `PENDING` streams that belong to the same + // `parent` table. + // + // Streams must be finalized before commit and cannot be committed multiple + // times. Once a stream is committed, data in the stream becomes available + // for read operations. + rpc BatchCommitWriteStreams(BatchCommitWriteStreamsRequest) returns (BatchCommitWriteStreamsResponse) { + option (google.api.http) = { + get: "/v1/{parent=projects/*/datasets/*/tables/*}" + }; + option (google.api.method_signature) = "parent"; + } + + // Flushes rows to a BUFFERED stream. + // + // If users are appending rows to BUFFERED stream, flush operation is + // required in order for the rows to become available for reading. A + // Flush operation flushes up to any previously flushed offset in a BUFFERED + // stream, to the offset specified in the request. + // + // Flush is not supported on the _default stream, since it is not BUFFERED. + rpc FlushRows(FlushRowsRequest) returns (FlushRowsResponse) { + option (google.api.http) = { + post: "/v1/{write_stream=projects/*/datasets/*/tables/*/streams/*}" + body: "*" + }; + option (google.api.method_signature) = "write_stream"; + } +} + // Request message for `CreateReadSession`. message CreateReadSessionRequest { // Required. The request project that owns the session, in the form of @@ -153,7 +273,7 @@ message ThrottleState { int32 throttle_percent = 1; } -// Estimated stream statistics for a given Stream. +// Estimated stream statistics for a given read Stream. message StreamStats { message Progress { // The fraction of rows assigned to the stream that have been processed by @@ -246,3 +366,242 @@ message SplitReadStreamResponse { // value indicates that the original stream can no longer be split. ReadStream remainder_stream = 2; } + +// Request message for `CreateWriteStream`. +message CreateWriteStreamRequest { + // Required. Reference to the table to which the stream belongs, in the format + // of `projects/{project}/datasets/{dataset}/tables/{table}`. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "bigquery.googleapis.com/Table" + } + ]; + + // Required. Stream to be created. + WriteStream write_stream = 2 [(google.api.field_behavior) = REQUIRED]; +} + +// Request message for `AppendRows`. +// +// Due to the nature of AppendRows being a bidirectional streaming RPC, certain +// parts of the AppendRowsRequest need only be specified for the first request +// sent each time the gRPC network connection is opened/reopened. +message AppendRowsRequest { + // ProtoData contains the data rows and schema when constructing append + // requests. + message ProtoData { + // Proto schema used to serialize the data. This value only needs to be + // provided as part of the first request on a gRPC network connection, + // and will be ignored for subsequent requests on the connection. + ProtoSchema writer_schema = 1; + + // Serialized row data in protobuf message format. + // Currently, the backend expects the serialized rows to adhere to + // proto2 semantics when appending rows, particularly with respect to + // how default values are encoded. + ProtoRows rows = 2; + } + + // Required. The write_stream identifies the target of the append operation, and only + // needs to be specified as part of the first request on the gRPC connection. + // If provided for subsequent requests, it must match the value of the first + // request. + // + // For explicitly created write streams, the format is: + // `projects/{project}/datasets/{dataset}/tables/{table}/streams/{id}` + // + // For the special default stream, the format is: + // `projects/{project}/datasets/{dataset}/tables/{table}/_default`. + string write_stream = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "bigquerystorage.googleapis.com/WriteStream" + } + ]; + + // If present, the write is only performed if the next append offset is same + // as the provided value. If not present, the write is performed at the + // current end of stream. Specifying a value for this field is not allowed + // when calling AppendRows for the '_default' stream. + google.protobuf.Int64Value offset = 2; + + // Input rows. The `writer_schema` field must be specified at the initial + // request and currently, it will be ignored if specified in following + // requests. Following requests must have data in the same format as the + // initial request. + oneof rows { + // Rows in proto format. + ProtoData proto_rows = 4; + } + + // Id set by client to annotate its identity. Only initial request setting is + // respected. + string trace_id = 6; +} + +// Response message for `AppendRows`. +message AppendRowsResponse { + // AppendResult is returned for successful append requests. + message AppendResult { + // The row offset at which the last append occurred. The offset will not be + // set if appending using default streams. + google.protobuf.Int64Value offset = 1; + } + + oneof response { + // Result if the append is successful. + AppendResult append_result = 1; + + // Error returned when problems were encountered. If present, + // it indicates rows were not accepted into the system. + // Users can retry or continue with other append requests within the + // same connection. + // + // Additional information about error signalling: + // + // ALREADY_EXISTS: Happens when an append specified an offset, and the + // backend already has received data at this offset. Typically encountered + // in retry scenarios, and can be ignored. + // + // OUT_OF_RANGE: Returned when the specified offset in the stream is beyond + // the current end of the stream. + // + // INVALID_ARGUMENT: Indicates a malformed request or data. + // + // ABORTED: Request processing is aborted because of prior failures. The + // request can be retried if previous failure is addressed. + // + // INTERNAL: Indicates server side error(s) that can be retried. + google.rpc.Status error = 2; + } + + // If backend detects a schema update, pass it to user so that user can + // use it to input new type of message. It will be empty when no schema + // updates have occurred. + TableSchema updated_schema = 3; +} + +// Request message for `GetWriteStreamRequest`. +message GetWriteStreamRequest { + // Required. Name of the stream to get, in the form of + // `projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}`. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "bigquerystorage.googleapis.com/WriteStream" + } + ]; +} + +// Request message for `BatchCommitWriteStreams`. +message BatchCommitWriteStreamsRequest { + // Required. Parent table that all the streams should belong to, in the form of + // `projects/{project}/datasets/{dataset}/tables/{table}`. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED + ]; + + // Required. The group of streams that will be committed atomically. + repeated string write_streams = 2 [(google.api.field_behavior) = REQUIRED]; +} + +// Response message for `BatchCommitWriteStreams`. +message BatchCommitWriteStreamsResponse { + // The time at which streams were committed in microseconds granularity. + // This field will only exist when there are no stream errors. + // **Note** if this field is not set, it means the commit was not successful. + google.protobuf.Timestamp commit_time = 1; + + // Stream level error if commit failed. Only streams with error will be in + // the list. + // If empty, there is no error and all streams are committed successfully. + // If non empty, certain streams have errors and ZERO stream is committed due + // to atomicity guarantee. + repeated StorageError stream_errors = 2; +} + +// Request message for invoking `FinalizeWriteStream`. +message FinalizeWriteStreamRequest { + // Required. Name of the stream to finalize, in the form of + // `projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}`. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "bigquerystorage.googleapis.com/WriteStream" + } + ]; +} + +// Response message for `FinalizeWriteStream`. +message FinalizeWriteStreamResponse { + // Number of rows in the finalized stream. + int64 row_count = 1; +} + +// Request message for `FlushRows`. +message FlushRowsRequest { + // Required. The stream that is the target of the flush operation. + string write_stream = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "bigquerystorage.googleapis.com/WriteStream" + } + ]; + + // Ending offset of the flush operation. Rows before this offset(including + // this offset) will be flushed. + google.protobuf.Int64Value offset = 2; +} + +// Respond message for `FlushRows`. +message FlushRowsResponse { + // The rows before this offset (including this offset) are flushed. + int64 offset = 1; +} + +// Structured custom BigQuery Storage error message. The error can be attached +// as error details in the returned rpc Status. In particular, the use of error +// codes allows more structured error handling, and reduces the need to evaluate +// unstructured error text strings. +message StorageError { + // Error code for `StorageError`. + enum StorageErrorCode { + // Default error. + STORAGE_ERROR_CODE_UNSPECIFIED = 0; + + // Table is not found in the system. + TABLE_NOT_FOUND = 1; + + // Stream is already committed. + STREAM_ALREADY_COMMITTED = 2; + + // Stream is not found. + STREAM_NOT_FOUND = 3; + + // Invalid Stream type. + // For example, you try to commit a stream that is not pending. + INVALID_STREAM_TYPE = 4; + + // Invalid Stream state. + // For example, you try to commit a stream that is not finalized or is + // garbaged. + INVALID_STREAM_STATE = 5; + + // Stream is finalized. + STREAM_FINALIZED = 6; + + // There is a schema mismatch and it is caused by user schema has extra + // field than bigquery schema. + SCHEMA_MISMATCH_EXTRA_FIELDS = 7; + } + + // BigQuery Storage specific error code. + StorageErrorCode code = 1; + + // Name of the failed entity. + string entity = 2; + + // Message that describes the error. + string error_message = 3; +} diff --git a/google/cloud/bigquery/storage/v1/stream.proto b/google/cloud/bigquery/storage/v1/stream.proto index 51aeaac465..dc62e8c00c 100644 --- a/google/cloud/bigquery/storage/v1/stream.proto +++ b/google/cloud/bigquery/storage/v1/stream.proto @@ -20,6 +20,7 @@ import "google/api/field_behavior.proto"; import "google/api/resource.proto"; import "google/cloud/bigquery/storage/v1/arrow.proto"; import "google/cloud/bigquery/storage/v1/avro.proto"; +import "google/cloud/bigquery/storage/v1/table.proto"; import "google/protobuf/timestamp.proto"; option csharp_namespace = "Google.Cloud.BigQuery.Storage.V1"; @@ -146,3 +147,50 @@ message ReadStream { // `projects/{project_id}/locations/{location}/sessions/{session_id}/streams/{stream_id}`. string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; } + +// Information about a single stream that gets data inside the storage system. +message WriteStream { + option (google.api.resource) = { + type: "bigquerystorage.googleapis.com/WriteStream" + pattern: "projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}" + }; + + // Type enum of the stream. + enum Type { + // Unknown type. + TYPE_UNSPECIFIED = 0; + + // Data will commit automatically and appear as soon as the write is + // acknowledged. + COMMITTED = 1; + + // Data is invisible until the stream is committed. + PENDING = 2; + + // Data is only visible up to the offset to which it was flushed. + BUFFERED = 3; + } + + // Output only. Name of the stream, in the form + // `projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}`. + string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Immutable. Type of the stream. + Type type = 2 [(google.api.field_behavior) = IMMUTABLE]; + + // Output only. Create time of the stream. For the _default stream, this is the + // creation_time of the table. + google.protobuf.Timestamp create_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Commit time of the stream. + // If a stream is of `COMMITTED` type, then it will have a commit_time same as + // `create_time`. If the stream is of `PENDING` type, empty commit_time + // means it is not committed. + google.protobuf.Timestamp commit_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The schema of the destination table. It is only returned in + // `CreateWriteStream` response. Caller should generate data that's + // compatible with this schema to send in initial `AppendRowsRequest`. + // The table schema could go out of date during the life time of the stream. + TableSchema table_schema = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; +} diff --git a/google/cloud/bigquery/storage/v1/table.proto b/google/cloud/bigquery/storage/v1/table.proto new file mode 100644 index 0000000000..a8c6f844df --- /dev/null +++ b/google/cloud/bigquery/storage/v1/table.proto @@ -0,0 +1,164 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.cloud.bigquery.storage.v1; + +import "google/api/field_behavior.proto"; + +option csharp_namespace = "Google.Cloud.BigQuery.Storage.V1"; +option go_package = "google.golang.org/genproto/googleapis/cloud/bigquery/storage/v1;storage"; +option java_multiple_files = true; +option java_outer_classname = "TableProto"; +option java_package = "com.google.cloud.bigquery.storage.v1"; +option php_namespace = "Google\\Cloud\\BigQuery\\Storage\\V1"; + +// Schema of a table. +message TableSchema { + // Describes the fields in a table. + repeated TableFieldSchema fields = 1; +} + +// TableFieldSchema defines a single field/column within a table schema. +message TableFieldSchema { + enum Type { + // Illegal value + TYPE_UNSPECIFIED = 0; + + // 64K, UTF8 + STRING = 1; + + // 64-bit signed + INT64 = 2; + + // 64-bit IEEE floating point + DOUBLE = 3; + + // Aggregate type + STRUCT = 4; + + // 64K, Binary + BYTES = 5; + + // 2-valued + BOOL = 6; + + // 64-bit signed usec since UTC epoch + TIMESTAMP = 7; + + // Civil date - Year, Month, Day + DATE = 8; + + // Civil time - Hour, Minute, Second, Microseconds + TIME = 9; + + // Combination of civil date and civil time + DATETIME = 10; + + // Geography object + GEOGRAPHY = 11; + + // Numeric value + NUMERIC = 12; + + // BigNumeric value + BIGNUMERIC = 13; + + // Interval + INTERVAL = 14; + + // JSON, String + JSON = 15; + } + + enum Mode { + // Illegal value + MODE_UNSPECIFIED = 0; + + NULLABLE = 1; + + REQUIRED = 2; + + REPEATED = 3; + } + + // Required. The field name. The name must contain only letters (a-z, A-Z), + // numbers (0-9), or underscores (_), and must start with a letter or + // underscore. The maximum length is 128 characters. + string name = 1 [(google.api.field_behavior) = REQUIRED]; + + // Required. The field data type. + Type type = 2 [(google.api.field_behavior) = REQUIRED]; + + // Optional. The field mode. The default value is NULLABLE. + Mode mode = 3 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Describes the nested schema fields if the type property is set to STRUCT. + repeated TableFieldSchema fields = 4 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The field description. The maximum length is 1,024 characters. + string description = 6 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Maximum length of values of this field for STRINGS or BYTES. + // + // If max_length is not specified, no maximum length constraint is imposed + // on this field. + // + // If type = "STRING", then max_length represents the maximum UTF-8 + // length of strings in this field. + // + // If type = "BYTES", then max_length represents the maximum number of + // bytes in this field. + // + // It is invalid to set this field if type is not "STRING" or "BYTES". + int64 max_length = 7 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Precision (maximum number of total digits in base 10) and scale + // (maximum number of digits in the fractional part in base 10) constraints + // for values of this field for NUMERIC or BIGNUMERIC. + // + // It is invalid to set precision or scale if type is not "NUMERIC" or + // "BIGNUMERIC". + // + // If precision and scale are not specified, no value range constraint is + // imposed on this field insofar as values are permitted by the type. + // + // Values of this NUMERIC or BIGNUMERIC field must be in this range when: + // + // * Precision (P) and scale (S) are specified: + // [-10^(P-S) + 10^(-S), 10^(P-S) - 10^(-S)] + // * Precision (P) is specified but not scale (and thus scale is + // interpreted to be equal to zero): + // [-10^P + 1, 10^P - 1]. + // + // Acceptable values for precision and scale if both are specified: + // + // * If type = "NUMERIC": + // 1 <= precision - scale <= 29 and 0 <= scale <= 9. + // * If type = "BIGNUMERIC": + // 1 <= precision - scale <= 38 and 0 <= scale <= 38. + // + // Acceptable values for precision if only precision is specified but not + // scale (and thus scale is interpreted to be equal to zero): + // + // * If type = "NUMERIC": 1 <= precision <= 29. + // * If type = "BIGNUMERIC": 1 <= precision <= 38. + // + // If scale is specified but not precision, then it is invalid. + int64 precision = 8 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. See documentation for precision. + int64 scale = 9 [(google.api.field_behavior) = OPTIONAL]; +}