GlcoinCore/googleapis
1
0
Fork 0
mirror of https://github.com/googleapis/googleapis.git synced 2026-08-17 13:06:47 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: add BigQuery Storage Write API v1

Browse source 
Committer: @yirutang
PiperOrigin-RevId: 397350004
This commit is contained in:
Google APIs 2021-09-17 10:32:22 -07:00 • committed by Copybara-Service
parent ddf396f7e7
commit b4da4fd5ee
7 changed files with 726 additions and 49 deletions
Download patch file Download diff file Expand all files Collapse all files

95
google/cloud/bigquery/storage/v1/BUILD.bazel
View file

@ -12,15 +12,21 @@ proto_library(
srcs = [
"arrow.proto",
"avro.proto",
"protobuf.proto",
"storage.proto",
"stream.proto",
"table.proto",
],
deps = [
"//google/api:annotations_proto",
"//google/api:client_proto",
"//google/api:field_behavior_proto",
"//google/api:resource_proto",
"//google/rpc:status_proto",
"@com_google_protobuf//:descriptor_proto",
"@com_google_protobuf//:empty_proto",
"@com_google_protobuf//:timestamp_proto",
"@com_google_protobuf//:wrappers_proto",
],
)
@ -105,6 +111,7 @@ go_proto_library(
protos = [":storage_proto"],
deps = [
"//google/api:annotations_go_proto",
"//google/rpc:status_go_proto",
],
)
@ -236,53 +243,55 @@ nodejs_gapic_assembly_pkg(
##############################################################################
# Ruby
##############################################################################
load(
"@com_google_googleapis_imports//:imports.bzl",
"ruby_cloud_gapic_library",
"ruby_gapic_assembly_pkg",
"ruby_grpc_library",
"ruby_proto_library",
)
# disabled due to https://github.com/protocolbuffers/protobuf/issues/7992
#
# load(
# "@com_google_googleapis_imports//:imports.bzl",
# "ruby_cloud_gapic_library",
# "ruby_gapic_assembly_pkg",
# "ruby_grpc_library",
# "ruby_proto_library",
# )
ruby_proto_library(
name = "storage_ruby_proto",
deps = [":storage_proto"],
)
# ruby_proto_library(
# name = "storage_ruby_proto",
# deps = [":storage_proto"],
# )
ruby_grpc_library(
name = "storage_ruby_grpc",
srcs = [":storage_proto"],
deps = [":storage_ruby_proto"],
)
# ruby_grpc_library(
# name = "storage_ruby_grpc",
# srcs = [":storage_proto"],
# deps = [":storage_ruby_proto"],
# )
ruby_cloud_gapic_library(
name = "storage_ruby_gapic",
srcs = [":storage_proto_with_info"],
extra_protoc_parameters = [
"ruby-cloud-gem-name=google-cloud-bigquery-storage-v1",
"ruby-cloud-env-prefix=BIGQUERY_STORAGE",
"ruby-cloud-product-url=https://cloud.google.com/bigquery/docs/reference/storage",
"ruby-cloud-api-id=bigquerystorage.googleapis.com",
"ruby-cloud-api-shortname=bigquerystorage",
],
grpc_service_config = "bigquerystorage_grpc_service_config.json",
ruby_cloud_description = "The BigQuery Storage API provides fast access to BigQuery managed storage.",
ruby_cloud_title = "BigQuery Storage V1",
deps = [
":storage_ruby_grpc",
":storage_ruby_proto",
],
)
# ruby_cloud_gapic_library(
# name = "storage_ruby_gapic",
# srcs = [":storage_proto_with_info"],
# extra_protoc_parameters = [
# "ruby-cloud-gem-name=google-cloud-bigquery-storage-v1",
# "ruby-cloud-env-prefix=BIGQUERY_STORAGE",
# "ruby-cloud-product-url=https://cloud.google.com/bigquery/docs/reference/storage",
# "ruby-cloud-api-id=bigquerystorage.googleapis.com",
# "ruby-cloud-api-shortname=bigquerystorage",
# ],
# grpc_service_config = "bigquerystorage_grpc_service_config.json",
# ruby_cloud_description = "The BigQuery Storage API provides fast access to BigQuery managed storage.",
# ruby_cloud_title = "BigQuery Storage V1",
# deps = [
# ":storage_ruby_grpc",
# ":storage_ruby_proto",
# ],
# )
# Open Source Packages
ruby_gapic_assembly_pkg(
name = "google-cloud-bigquery-storage-v1-ruby",
deps = [
":storage_ruby_gapic",
":storage_ruby_grpc",
":storage_ruby_proto",
],
)
# # Open Source Packages
# ruby_gapic_assembly_pkg(
# name = "google-cloud-bigquery-storage-v1-ruby",
# deps = [
# ":storage_ruby_gapic",
# ":storage_ruby_grpc",
# ":storage_ruby_proto",
# ],
# )
##############################################################################
# C#

39
google/cloud/bigquery/storage/v1/bigquerystorage_grpc_service_config.json
View file

@ -34,12 +34,27 @@
"UNAVAILABLE"
]
}
},
{
}, {
"name": [
{
"service": "google.cloud.bigquery.storage.v1.BigQueryRead",
"method": "SplitReadStream"
"service": "google.cloud.bigquery.storage.v1.BigQueryWrite",
"method": "CreateWriteStream"
},
{
"service": "google.cloud.bigquery.storage.v1.BigQueryWrite",
"method": "BatchCommitWriteStreams"
},
{
"service": "google.cloud.bigquery.storage.v1.BigQueryWrite",
"method": "FinalizeWriteStream"
},
{
"service": "google.cloud.bigquery.storage.v1.BigQueryWrite",
"method": "GetWriteStream"
},
{
"service": "google.cloud.bigquery.storage.v1.BigQueryWrite",
"method": "FlushRows"
}
],
"timeout": "600s",
@ -52,6 +67,22 @@
"UNAVAILABLE"
]
}
}, {
"name": [
{
"service": "google.cloud.bigquery.storage.v1.BigQueryWrite",
"method": "AppendRows"
}
],
"timeout": "86400s",
"retryPolicy": {
"initialBackoff": "0.100s",
"maxBackoff": "60s",
"backoffMultiplier": 1.3,
"retryableStatusCodes": [
"UNAVAILABLE"
]
}
}
]
}

20
google/cloud/bigquery/storage/v1/bigquerystorage_v1.yaml
View file

@ -5,6 +5,7 @@ title: BigQuery Storage API
apis:
- name: google.cloud.bigquery.storage.v1.BigQueryRead
- name: google.cloud.bigquery.storage.v1.BigQueryWrite
backend:
rules:
@ -14,6 +15,18 @@ backend:
deadline: 21600.0
- selector: google.cloud.bigquery.storage.v1.BigQueryRead.SplitReadStream
deadline: 120.0
- selector: google.cloud.bigquery.storage.v1.BigQueryWrite.CreateWriteStram
deadline: 120.0
- selector: google.cloud.bigquery.storage.v1.BigQueryWrite.FinalizeWriteStream
deadline: 120.0
- selector: google.cloud.bigquery.storage.v1.BigQueryWrite.BatchCommitWriteStreams
deadline: 120.0
- selector: google.cloud.bigquery.storage.v1.BigQueryWrite.GetWriteStream
deadline: 120.0
- selector: google.cloud.bigquery.storage.v1.BigQueryWrite.FlushRows
deadline: 120.0
- selector: google.cloud.bigquery.storage.v1.BigQueryWrite.AppendRows
deadline: 86400.0
authentication:
rules:
@ -21,5 +34,10 @@ authentication:
oauth:
canonical_scopes: |-
https://www.googleapis.com/auth/bigquery,
https://www.googleapis.com/auth/bigquery.readonly,
https://www.googleapis.com/auth/cloud-platform
- selector: 'google.cloud.bigquery.storage.v1.BigQueryWrite.*'
oauth:
canonical_scopes: |-
https://www.googleapis.com/auth/bigquery,
https://www.googleapis.com/auth/bigquery.insertdata,
https://www.googleapis.com/auth/cloud-platform

48
google/cloud/bigquery/storage/v1/protobuf.proto Normal file
View file

@ -0,0 +1,48 @@
// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
package google.cloud.bigquery.storage.v1;
import "google/protobuf/descriptor.proto";
option csharp_namespace = "Google.Cloud.BigQuery.Storage.V1";
option go_package = "google.golang.org/genproto/googleapis/cloud/bigquery/storage/v1;storage";
option java_multiple_files = true;
option java_outer_classname = "ProtoBufProto";
option java_package = "com.google.cloud.bigquery.storage.v1";
option php_namespace = "Google\\Cloud\\BigQuery\\Storage\\V1";
// ProtoSchema describes the schema of the serialized protocol buffer data rows.
message ProtoSchema {
// Descriptor for input message. The provided descriptor must be self
// contained, such that data rows sent can be fully decoded using only the
// single descriptor. For data rows that are compositions of multiple
// independent messages, this means the descriptor may need to be transformed
// to only use nested types:
// https://developers.google.com/protocol-buffers/docs/proto#nested
//
// For additional information for how proto types and values map onto BigQuery
// see: https://cloud.google.com/bigquery/docs/write-api#data_type_conversions
google.protobuf.DescriptorProto proto_descriptor = 1;
}
message ProtoRows {
// A sequence of rows serialized as a Protocol Buffer.
//
// See https://developers.google.com/protocol-buffers/docs/overview for more
// information on deserializing this field.
repeated bytes serialized_rows = 1;
}

361
google/cloud/bigquery/storage/v1/storage.proto
View file

@ -22,7 +22,12 @@ import "google/api/field_behavior.proto";
import "google/api/resource.proto";
import "google/cloud/bigquery/storage/v1/arrow.proto";
import "google/cloud/bigquery/storage/v1/avro.proto";
import "google/cloud/bigquery/storage/v1/protobuf.proto";
import "google/cloud/bigquery/storage/v1/stream.proto";
import "google/cloud/bigquery/storage/v1/table.proto";
import "google/protobuf/timestamp.proto";
import "google/protobuf/wrappers.proto";
import "google/rpc/status.proto";
option csharp_namespace = "Google.Cloud.BigQuery.Storage.V1";
option go_package = "google.golang.org/genproto/googleapis/cloud/bigquery/storage/v1;storage";
@ -105,6 +110,121 @@ service BigQueryRead {
}
}
// BigQuery Write API.
//
// The Write API can be used to write data to BigQuery.
//
// For supplementary information about the Write API, see:
// https://cloud.google.com/bigquery/docs/write-api
service BigQueryWrite {
option (google.api.default_host) = "bigquerystorage.googleapis.com";
option (google.api.oauth_scopes) =
"https://www.googleapis.com/auth/bigquery,"
"https://www.googleapis.com/auth/bigquery.insertdata,"
"https://www.googleapis.com/auth/cloud-platform";
// Creates a write stream to the given table.
// Additionally, every table has a special stream named '_default'
// to which data can be written. This stream doesn't need to be created using
// CreateWriteStream. It is a stream that can be used simultaneously by any
// number of clients. Data written to this stream is considered committed as
// soon as an acknowledgement is received.
rpc CreateWriteStream(CreateWriteStreamRequest) returns (WriteStream) {
option (google.api.http) = {
post: "/v1/{parent=projects/*/datasets/*/tables/*}"
body: "write_stream"
};
option (google.api.method_signature) = "parent,write_stream";
}
// Appends data to the given stream.
//
// If `offset` is specified, the `offset` is checked against the end of
// stream. The server returns `OUT_OF_RANGE` in `AppendRowsResponse` if an
// attempt is made to append to an offset beyond the current end of the stream
// or `ALREADY_EXISTS` if user provides an `offset` that has already been
// written to. User can retry with adjusted offset within the same RPC
// connection. If `offset` is not specified, append happens at the end of the
// stream.
//
// The response contains an optional offset at which the append
// happened. No offset information will be returned for appends to a
// default stream.
//
// Responses are received in the same order in which requests are sent.
// There will be one response for each successful inserted request. Responses
// may optionally embed error information if the originating AppendRequest was
// not successfully processed.
//
// The specifics of when successfully appended data is made visible to the
// table are governed by the type of stream:
//
// * For COMMITTED streams (which includes the default stream), data is
// visible immediately upon successful append.
//
// * For BUFFERED streams, data is made visible via a subsequent `FlushRows`
// rpc which advances a cursor to a newer offset in the stream.
//
// * For PENDING streams, data is not made visible until the stream itself is
// finalized (via the `FinalizeWriteStream` rpc), and the stream is explicitly
// committed via the `BatchCommitWriteStreams` rpc.
rpc AppendRows(stream AppendRowsRequest) returns (stream AppendRowsResponse) {
option (google.api.http) = {
post: "/v1/{write_stream=projects/*/datasets/*/tables/*/streams/*}"
body: "*"
};
option (google.api.method_signature) = "write_stream";
}
// Gets information about a write stream.
rpc GetWriteStream(GetWriteStreamRequest) returns (WriteStream) {
option (google.api.http) = {
post: "/v1/{name=projects/*/datasets/*/tables/*/streams/*}"
body: "*"
};
option (google.api.method_signature) = "name";
}
// Finalize a write stream so that no new data can be appended to the
// stream. Finalize is not supported on the '_default' stream.
rpc FinalizeWriteStream(FinalizeWriteStreamRequest) returns (FinalizeWriteStreamResponse) {
option (google.api.http) = {
post: "/v1/{name=projects/*/datasets/*/tables/*/streams/*}"
body: "*"
};
option (google.api.method_signature) = "name";
}
// Atomically commits a group of `PENDING` streams that belong to the same
// `parent` table.
//
// Streams must be finalized before commit and cannot be committed multiple
// times. Once a stream is committed, data in the stream becomes available
// for read operations.
rpc BatchCommitWriteStreams(BatchCommitWriteStreamsRequest) returns (BatchCommitWriteStreamsResponse) {
option (google.api.http) = {
get: "/v1/{parent=projects/*/datasets/*/tables/*}"
};
option (google.api.method_signature) = "parent";
}
// Flushes rows to a BUFFERED stream.
//
// If users are appending rows to BUFFERED stream, flush operation is
// required in order for the rows to become available for reading. A
// Flush operation flushes up to any previously flushed offset in a BUFFERED
// stream, to the offset specified in the request.
//
// Flush is not supported on the _default stream, since it is not BUFFERED.
rpc FlushRows(FlushRowsRequest) returns (FlushRowsResponse) {
option (google.api.http) = {
post: "/v1/{write_stream=projects/*/datasets/*/tables/*/streams/*}"
body: "*"
};
option (google.api.method_signature) = "write_stream";
}
}
// Request message for `CreateReadSession`.
message CreateReadSessionRequest {
// Required. The request project that owns the session, in the form of
@ -153,7 +273,7 @@ message ThrottleState {
int32 throttle_percent = 1;
}
// Estimated stream statistics for a given Stream.
// Estimated stream statistics for a given read Stream.
message StreamStats {
message Progress {
// The fraction of rows assigned to the stream that have been processed by
@ -246,3 +366,242 @@ message SplitReadStreamResponse {
// value indicates that the original stream can no longer be split.
ReadStream remainder_stream = 2;
}
// Request message for `CreateWriteStream`.
message CreateWriteStreamRequest {
// Required. Reference to the table to which the stream belongs, in the format
// of `projects/{project}/datasets/{dataset}/tables/{table}`.
string parent = 1 [
(google.api.field_behavior) = REQUIRED,
(google.api.resource_reference) = {
type: "bigquery.googleapis.com/Table"
}
];
// Required. Stream to be created.
WriteStream write_stream = 2 [(google.api.field_behavior) = REQUIRED];
}
// Request message for `AppendRows`.
//
// Due to the nature of AppendRows being a bidirectional streaming RPC, certain
// parts of the AppendRowsRequest need only be specified for the first request
// sent each time the gRPC network connection is opened/reopened.
message AppendRowsRequest {
// ProtoData contains the data rows and schema when constructing append
// requests.
message ProtoData {
// Proto schema used to serialize the data. This value only needs to be
// provided as part of the first request on a gRPC network connection,
// and will be ignored for subsequent requests on the connection.
ProtoSchema writer_schema = 1;
// Serialized row data in protobuf message format.
// Currently, the backend expects the serialized rows to adhere to
// proto2 semantics when appending rows, particularly with respect to
// how default values are encoded.
ProtoRows rows = 2;
}
// Required. The write_stream identifies the target of the append operation, and only
// needs to be specified as part of the first request on the gRPC connection.
// If provided for subsequent requests, it must match the value of the first
// request.
//
// For explicitly created write streams, the format is:
// `projects/{project}/datasets/{dataset}/tables/{table}/streams/{id}`
//
// For the special default stream, the format is:
// `projects/{project}/datasets/{dataset}/tables/{table}/_default`.
string write_stream = 1 [
(google.api.field_behavior) = REQUIRED,
(google.api.resource_reference) = {
type: "bigquerystorage.googleapis.com/WriteStream"
}
];
// If present, the write is only performed if the next append offset is same
// as the provided value. If not present, the write is performed at the
// current end of stream. Specifying a value for this field is not allowed
// when calling AppendRows for the '_default' stream.
google.protobuf.Int64Value offset = 2;
// Input rows. The `writer_schema` field must be specified at the initial
// request and currently, it will be ignored if specified in following
// requests. Following requests must have data in the same format as the
// initial request.
oneof rows {
// Rows in proto format.
ProtoData proto_rows = 4;
}
// Id set by client to annotate its identity. Only initial request setting is
// respected.
string trace_id = 6;
}
// Response message for `AppendRows`.
message AppendRowsResponse {
// AppendResult is returned for successful append requests.
message AppendResult {
// The row offset at which the last append occurred. The offset will not be
// set if appending using default streams.
google.protobuf.Int64Value offset = 1;
}
oneof response {
// Result if the append is successful.
AppendResult append_result = 1;
// Error returned when problems were encountered. If present,
// it indicates rows were not accepted into the system.
// Users can retry or continue with other append requests within the
// same connection.
//
// Additional information about error signalling:
//
// ALREADY_EXISTS: Happens when an append specified an offset, and the
// backend already has received data at this offset. Typically encountered
// in retry scenarios, and can be ignored.
//
// OUT_OF_RANGE: Returned when the specified offset in the stream is beyond
// the current end of the stream.
//
// INVALID_ARGUMENT: Indicates a malformed request or data.
//
// ABORTED: Request processing is aborted because of prior failures. The
// request can be retried if previous failure is addressed.
//
// INTERNAL: Indicates server side error(s) that can be retried.
google.rpc.Status error = 2;
}
// If backend detects a schema update, pass it to user so that user can
// use it to input new type of message. It will be empty when no schema
// updates have occurred.
TableSchema updated_schema = 3;
}
// Request message for `GetWriteStreamRequest`.
message GetWriteStreamRequest {
// Required. Name of the stream to get, in the form of
// `projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}`.
string name = 1 [
(google.api.field_behavior) = REQUIRED,
(google.api.resource_reference) = {
type: "bigquerystorage.googleapis.com/WriteStream"
}
];
}
// Request message for `BatchCommitWriteStreams`.
message BatchCommitWriteStreamsRequest {
// Required. Parent table that all the streams should belong to, in the form of
// `projects/{project}/datasets/{dataset}/tables/{table}`.
string parent = 1 [
(google.api.field_behavior) = REQUIRED
];
// Required. The group of streams that will be committed atomically.
repeated string write_streams = 2 [(google.api.field_behavior) = REQUIRED];
}
// Response message for `BatchCommitWriteStreams`.
message BatchCommitWriteStreamsResponse {
// The time at which streams were committed in microseconds granularity.
// This field will only exist when there are no stream errors.
// **Note** if this field is not set, it means the commit was not successful.
google.protobuf.Timestamp commit_time = 1;
// Stream level error if commit failed. Only streams with error will be in
// the list.
// If empty, there is no error and all streams are committed successfully.
// If non empty, certain streams have errors and ZERO stream is committed due
// to atomicity guarantee.
repeated StorageError stream_errors = 2;
}
// Request message for invoking `FinalizeWriteStream`.
message FinalizeWriteStreamRequest {
// Required. Name of the stream to finalize, in the form of
// `projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}`.
string name = 1 [
(google.api.field_behavior) = REQUIRED,
(google.api.resource_reference) = {
type: "bigquerystorage.googleapis.com/WriteStream"
}
];
}
// Response message for `FinalizeWriteStream`.
message FinalizeWriteStreamResponse {
// Number of rows in the finalized stream.
int64 row_count = 1;
}
// Request message for `FlushRows`.
message FlushRowsRequest {
// Required. The stream that is the target of the flush operation.
string write_stream = 1 [
(google.api.field_behavior) = REQUIRED,
(google.api.resource_reference) = {
type: "bigquerystorage.googleapis.com/WriteStream"
}
];
// Ending offset of the flush operation. Rows before this offset(including
// this offset) will be flushed.
google.protobuf.Int64Value offset = 2;
}
// Respond message for `FlushRows`.
message FlushRowsResponse {
// The rows before this offset (including this offset) are flushed.
int64 offset = 1;
}
// Structured custom BigQuery Storage error message. The error can be attached
// as error details in the returned rpc Status. In particular, the use of error
// codes allows more structured error handling, and reduces the need to evaluate
// unstructured error text strings.
message StorageError {
// Error code for `StorageError`.
enum StorageErrorCode {
// Default error.
STORAGE_ERROR_CODE_UNSPECIFIED = 0;
// Table is not found in the system.
TABLE_NOT_FOUND = 1;
// Stream is already committed.
STREAM_ALREADY_COMMITTED = 2;
// Stream is not found.
STREAM_NOT_FOUND = 3;
// Invalid Stream type.
// For example, you try to commit a stream that is not pending.
INVALID_STREAM_TYPE = 4;
// Invalid Stream state.
// For example, you try to commit a stream that is not finalized or is
// garbaged.
INVALID_STREAM_STATE = 5;
// Stream is finalized.
STREAM_FINALIZED = 6;
// There is a schema mismatch and it is caused by user schema has extra
// field than bigquery schema.
SCHEMA_MISMATCH_EXTRA_FIELDS = 7;
}
// BigQuery Storage specific error code.
StorageErrorCode code = 1;
// Name of the failed entity.
string entity = 2;
// Message that describes the error.
string error_message = 3;
}

48
google/cloud/bigquery/storage/v1/stream.proto
View file

@ -20,6 +20,7 @@ import "google/api/field_behavior.proto";
import "google/api/resource.proto";
import "google/cloud/bigquery/storage/v1/arrow.proto";
import "google/cloud/bigquery/storage/v1/avro.proto";
import "google/cloud/bigquery/storage/v1/table.proto";
import "google/protobuf/timestamp.proto";
option csharp_namespace = "Google.Cloud.BigQuery.Storage.V1";
@ -146,3 +147,50 @@ message ReadStream {
// `projects/{project_id}/locations/{location}/sessions/{session_id}/streams/{stream_id}`.
string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
}
// Information about a single stream that gets data inside the storage system.
message WriteStream {
option (google.api.resource) = {
type: "bigquerystorage.googleapis.com/WriteStream"
pattern: "projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}"
};
// Type enum of the stream.
enum Type {
// Unknown type.
TYPE_UNSPECIFIED = 0;
// Data will commit automatically and appear as soon as the write is
// acknowledged.
COMMITTED = 1;
// Data is invisible until the stream is committed.
PENDING = 2;
// Data is only visible up to the offset to which it was flushed.
BUFFERED = 3;
}
// Output only. Name of the stream, in the form
// `projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}`.
string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
// Immutable. Type of the stream.
Type type = 2 [(google.api.field_behavior) = IMMUTABLE];
// Output only. Create time of the stream. For the _default stream, this is the
// creation_time of the table.
google.protobuf.Timestamp create_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. Commit time of the stream.
// If a stream is of `COMMITTED` type, then it will have a commit_time same as
// `create_time`. If the stream is of `PENDING` type, empty commit_time
// means it is not committed.
google.protobuf.Timestamp commit_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. The schema of the destination table. It is only returned in
// `CreateWriteStream` response. Caller should generate data that's
// compatible with this schema to send in initial `AppendRowsRequest`.
// The table schema could go out of date during the life time of the stream.
TableSchema table_schema = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
}

164
google/cloud/bigquery/storage/v1/table.proto Normal file
View file

@ -0,0 +1,164 @@
// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
package google.cloud.bigquery.storage.v1;
import "google/api/field_behavior.proto";
option csharp_namespace = "Google.Cloud.BigQuery.Storage.V1";
option go_package = "google.golang.org/genproto/googleapis/cloud/bigquery/storage/v1;storage";
option java_multiple_files = true;
option java_outer_classname = "TableProto";
option java_package = "com.google.cloud.bigquery.storage.v1";
option php_namespace = "Google\\Cloud\\BigQuery\\Storage\\V1";
// Schema of a table.
message TableSchema {
// Describes the fields in a table.
repeated TableFieldSchema fields = 1;
}
// TableFieldSchema defines a single field/column within a table schema.
message TableFieldSchema {
enum Type {
// Illegal value
TYPE_UNSPECIFIED = 0;
// 64K, UTF8
STRING = 1;
// 64-bit signed
INT64 = 2;
// 64-bit IEEE floating point
DOUBLE = 3;
// Aggregate type
STRUCT = 4;
// 64K, Binary
BYTES = 5;
// 2-valued
BOOL = 6;
// 64-bit signed usec since UTC epoch
TIMESTAMP = 7;
// Civil date - Year, Month, Day
DATE = 8;
// Civil time - Hour, Minute, Second, Microseconds
TIME = 9;
// Combination of civil date and civil time
DATETIME = 10;
// Geography object
GEOGRAPHY = 11;
// Numeric value
NUMERIC = 12;
// BigNumeric value
BIGNUMERIC = 13;
// Interval
INTERVAL = 14;
// JSON, String
JSON = 15;
}
enum Mode {
// Illegal value
MODE_UNSPECIFIED = 0;
NULLABLE = 1;
REQUIRED = 2;
REPEATED = 3;
}
// Required. The field name. The name must contain only letters (a-z, A-Z),
// numbers (0-9), or underscores (_), and must start with a letter or
// underscore. The maximum length is 128 characters.
string name = 1 [(google.api.field_behavior) = REQUIRED];
// Required. The field data type.
Type type = 2 [(google.api.field_behavior) = REQUIRED];
// Optional. The field mode. The default value is NULLABLE.
Mode mode = 3 [(google.api.field_behavior) = OPTIONAL];
// Optional. Describes the nested schema fields if the type property is set to STRUCT.
repeated TableFieldSchema fields = 4 [(google.api.field_behavior) = OPTIONAL];
// Optional. The field description. The maximum length is 1,024 characters.
string description = 6 [(google.api.field_behavior) = OPTIONAL];
// Optional. Maximum length of values of this field for STRINGS or BYTES.
//
// If max_length is not specified, no maximum length constraint is imposed
// on this field.
//
// If type = "STRING", then max_length represents the maximum UTF-8
// length of strings in this field.
//
// If type = "BYTES", then max_length represents the maximum number of
// bytes in this field.
//
// It is invalid to set this field if type is not "STRING" or "BYTES".
int64 max_length = 7 [(google.api.field_behavior) = OPTIONAL];
// Optional. Precision (maximum number of total digits in base 10) and scale
// (maximum number of digits in the fractional part in base 10) constraints
// for values of this field for NUMERIC or BIGNUMERIC.
//
// It is invalid to set precision or scale if type is not "NUMERIC" or
// "BIGNUMERIC".
//
// If precision and scale are not specified, no value range constraint is
// imposed on this field insofar as values are permitted by the type.
//
// Values of this NUMERIC or BIGNUMERIC field must be in this range when:
//
// * Precision (P) and scale (S) are specified:
// [-10^(P-S) + 10^(-S), 10^(P-S) - 10^(-S)]
// * Precision (P) is specified but not scale (and thus scale is
// interpreted to be equal to zero):
// [-10^P + 1, 10^P - 1].
//
// Acceptable values for precision and scale if both are specified:
//
// * If type = "NUMERIC":
// 1 <= precision - scale <= 29 and 0 <= scale <= 9.
// * If type = "BIGNUMERIC":
// 1 <= precision - scale <= 38 and 0 <= scale <= 38.
//
// Acceptable values for precision if only precision is specified but not
// scale (and thus scale is interpreted to be equal to zero):
//
// * If type = "NUMERIC": 1 <= precision <= 29.
// * If type = "BIGNUMERIC": 1 <= precision <= 38.
//
// If scale is specified but not precision, then it is invalid.
int64 precision = 8 [(google.api.field_behavior) = OPTIONAL];
// Optional. See documentation for precision.
int64 scale = 9 [(google.api.field_behavior) = OPTIONAL];
}