mirror of
https://github.com/googleapis/googleapis.git
synced 2026-08-16 13:00:34 +02:00
feat(dataplex): add MetadataFeed to CatalogService for tracking metadata changes feat(dataplex): add LookupContext to CatalogService for LLM-generated resource context feat(dataplex): add support for attaching aspects to EntryLinks feat(dataplex): add UpdateEntryLink and LookupEntryLinks methods to CatalogService feat(dataplex): support OneTime triggers for DataScan operations feat(dataplex): add debug query support to Data Quality rules feat(dataplex): allow selective generation scope for Data Documentation scans feat(dataplex): add SKIPPED state to DataScan catalog publishing status docs(dataplex): remove deprecated metadata change warnings in Dataplex Catalog The DataProductService provides APIs to curate and manage collections of data assets as data products, enabling more organized sharing and usage for specific business cases. MetadataFeeds allow users to monitor metadata changes (CREATE, UPDATE, DELETE) within a specified scope (organization, project, or entry group) and publish them to Pub/Sub. CatalogService now includes a LookupContext API to provide LLM-generated context for resources, and enhanced EntryLink management, including the ability to attach aspects. DataScan operations now support a OneTime trigger for single-run scans, and Data Quality rules support DebugQueries to help investigate rule failures by returning diagnostic values. fix(dataplex)!: remove deprecated Explore-related methods and messages from public client libraries Breaking Changes: - Removed ContentService and all associated methods (CreateContent, UpdateContent, DeleteContent, GetContent, ListContent, etc.) and request/response messages. - Removed Environment and Session management methods from DataplexService (CreateEnvironment, UpdateEnvironment, DeleteEnvironment, ListEnvironments, GetEnvironment, ListSessions) and their associated messages. PiperOrigin-RevId: 883306841
811 lines
30 KiB
Protocol Buffer
811 lines
30 KiB
Protocol Buffer
// Copyright 2026 Google LLC
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
syntax = "proto3";
|
|
|
|
package google.cloud.dataplex.v1;
|
|
|
|
import "google/api/field_behavior.proto";
|
|
import "google/api/resource.proto";
|
|
import "google/protobuf/duration.proto";
|
|
import "google/protobuf/timestamp.proto";
|
|
|
|
option go_package = "cloud.google.com/go/dataplex/apiv1/dataplexpb;dataplexpb";
|
|
option java_multiple_files = true;
|
|
option java_outer_classname = "ResourcesProto";
|
|
option java_package = "com.google.cloud.dataplex.v1";
|
|
|
|
// A lake is a centralized repository for managing enterprise data across the
|
|
// organization distributed across many cloud projects, and stored in a variety
|
|
// of storage services such as Google Cloud Storage and BigQuery. The resources
|
|
// attached to a lake are referred to as managed resources. Data within these
|
|
// managed resources can be structured or unstructured. A lake provides data
|
|
// admins with tools to organize, secure and manage their data at scale, and
|
|
// provides data scientists and data engineers an integrated experience to
|
|
// easily search, discover, analyze and transform data and associated metadata.
|
|
message Lake {
|
|
option (google.api.resource) = {
|
|
type: "dataplex.googleapis.com/Lake"
|
|
pattern: "projects/{project}/locations/{location}/lakes/{lake}"
|
|
};
|
|
|
|
// Settings to manage association of Dataproc Metastore with a lake.
|
|
message Metastore {
|
|
// Optional. A relative reference to the Dataproc Metastore
|
|
// (https://cloud.google.com/dataproc-metastore/docs) service associated
|
|
// with the lake:
|
|
// `projects/{project_id}/locations/{location_id}/services/{service_id}`
|
|
string service = 1 [(google.api.field_behavior) = OPTIONAL];
|
|
}
|
|
|
|
// Status of Lake and Dataproc Metastore service instance association.
|
|
message MetastoreStatus {
|
|
// Current state of association.
|
|
enum State {
|
|
// Unspecified.
|
|
STATE_UNSPECIFIED = 0;
|
|
|
|
// A Metastore service instance is not associated with the lake.
|
|
NONE = 1;
|
|
|
|
// A Metastore service instance is attached to the lake.
|
|
READY = 2;
|
|
|
|
// Attach/detach is in progress.
|
|
UPDATING = 3;
|
|
|
|
// Attach/detach could not be done due to errors.
|
|
ERROR = 4;
|
|
}
|
|
|
|
// Current state of association.
|
|
State state = 1;
|
|
|
|
// Additional information about the current status.
|
|
string message = 2;
|
|
|
|
// Last update time of the metastore status of the lake.
|
|
google.protobuf.Timestamp update_time = 3;
|
|
|
|
// The URI of the endpoint used to access the Metastore service.
|
|
string endpoint = 4;
|
|
}
|
|
|
|
// Output only. The relative resource name of the lake, of the form:
|
|
// `projects/{project_number}/locations/{location_id}/lakes/{lake_id}`.
|
|
string name = 1 [
|
|
(google.api.field_behavior) = OUTPUT_ONLY,
|
|
(google.api.resource_reference) = { type: "dataplex.googleapis.com/Lake" }
|
|
];
|
|
|
|
// Optional. User friendly display name.
|
|
string display_name = 2 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Output only. System generated globally unique ID for the lake. This ID will
|
|
// be different if the lake is deleted and re-created with the same name.
|
|
string uid = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
|
|
|
|
// Output only. The time when the lake was created.
|
|
google.protobuf.Timestamp create_time = 4
|
|
[(google.api.field_behavior) = OUTPUT_ONLY];
|
|
|
|
// Output only. The time when the lake was last updated.
|
|
google.protobuf.Timestamp update_time = 5
|
|
[(google.api.field_behavior) = OUTPUT_ONLY];
|
|
|
|
// Optional. User-defined labels for the lake.
|
|
map<string, string> labels = 6 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Description of the lake.
|
|
string description = 7 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Output only. Current state of the lake.
|
|
State state = 8 [(google.api.field_behavior) = OUTPUT_ONLY];
|
|
|
|
// Output only. Service account associated with this lake. This service
|
|
// account must be authorized to access or operate on resources managed by the
|
|
// lake.
|
|
string service_account = 9 [(google.api.field_behavior) = OUTPUT_ONLY];
|
|
|
|
// Optional. Settings to manage lake and Dataproc Metastore service instance
|
|
// association.
|
|
Metastore metastore = 102 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Output only. Aggregated status of the underlying assets of the lake.
|
|
AssetStatus asset_status = 103 [(google.api.field_behavior) = OUTPUT_ONLY];
|
|
|
|
// Output only. Metastore status of the lake.
|
|
MetastoreStatus metastore_status = 104
|
|
[(google.api.field_behavior) = OUTPUT_ONLY];
|
|
}
|
|
|
|
// Aggregated status of the underlying assets of a lake or zone.
|
|
message AssetStatus {
|
|
// Last update time of the status.
|
|
google.protobuf.Timestamp update_time = 1;
|
|
|
|
// Number of active assets.
|
|
int32 active_assets = 2;
|
|
|
|
// Number of assets that are in process of updating the security policy on
|
|
// attached resources.
|
|
int32 security_policy_applying_assets = 3;
|
|
}
|
|
|
|
// A zone represents a logical group of related assets within a lake. A zone can
|
|
// be used to map to organizational structure or represent stages of data
|
|
// readiness from raw to curated. It provides managing behavior that is shared
|
|
// or inherited by all contained assets.
|
|
message Zone {
|
|
option (google.api.resource) = {
|
|
type: "dataplex.googleapis.com/Zone"
|
|
pattern: "projects/{project}/locations/{location}/lakes/{lake}/zones/{zone}"
|
|
};
|
|
|
|
// Type of zone.
|
|
enum Type {
|
|
// Zone type not specified.
|
|
TYPE_UNSPECIFIED = 0;
|
|
|
|
// A zone that contains data that needs further processing before it is
|
|
// considered generally ready for consumption and analytics workloads.
|
|
RAW = 1;
|
|
|
|
// A zone that contains data that is considered to be ready for broader
|
|
// consumption and analytics workloads. Curated structured data stored in
|
|
// Cloud Storage must conform to certain file formats (parquet, avro and
|
|
// orc) and organized in a hive-compatible directory layout.
|
|
CURATED = 2;
|
|
}
|
|
|
|
// Settings for resources attached as assets within a zone.
|
|
message ResourceSpec {
|
|
// Location type of the resources attached to a zone.
|
|
enum LocationType {
|
|
// Unspecified location type.
|
|
LOCATION_TYPE_UNSPECIFIED = 0;
|
|
|
|
// Resources that are associated with a single region.
|
|
SINGLE_REGION = 1;
|
|
|
|
// Resources that are associated with a multi-region location.
|
|
MULTI_REGION = 2;
|
|
}
|
|
|
|
// Required. Immutable. The location type of the resources that are allowed
|
|
// to be attached to the assets within this zone.
|
|
LocationType location_type = 1 [
|
|
(google.api.field_behavior) = REQUIRED,
|
|
(google.api.field_behavior) = IMMUTABLE
|
|
];
|
|
}
|
|
|
|
// Settings to manage the metadata discovery and publishing in a zone.
|
|
message DiscoverySpec {
|
|
// Describe CSV and similar semi-structured data formats.
|
|
message CsvOptions {
|
|
// Optional. The number of rows to interpret as header rows that should be
|
|
// skipped when reading data rows.
|
|
int32 header_rows = 1 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. The delimiter being used to separate values. This defaults to
|
|
// ','.
|
|
string delimiter = 2 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. The character encoding of the data. The default is UTF-8.
|
|
string encoding = 3 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Whether to disable the inference of data type for CSV data.
|
|
// If true, all columns will be registered as strings.
|
|
bool disable_type_inference = 4 [(google.api.field_behavior) = OPTIONAL];
|
|
}
|
|
|
|
// Describe JSON data format.
|
|
message JsonOptions {
|
|
// Optional. The character encoding of the data. The default is UTF-8.
|
|
string encoding = 1 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Whether to disable the inference of data type for Json data.
|
|
// If true, all columns will be registered as their primitive types
|
|
// (strings, number or boolean).
|
|
bool disable_type_inference = 2 [(google.api.field_behavior) = OPTIONAL];
|
|
}
|
|
|
|
// Required. Whether discovery is enabled.
|
|
bool enabled = 1 [(google.api.field_behavior) = REQUIRED];
|
|
|
|
// Optional. The list of patterns to apply for selecting data to include
|
|
// during discovery if only a subset of the data should considered. For
|
|
// Cloud Storage bucket assets, these are interpreted as glob patterns used
|
|
// to match object names. For BigQuery dataset assets, these are interpreted
|
|
// as patterns to match table names.
|
|
repeated string include_patterns = 2
|
|
[(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. The list of patterns to apply for selecting data to exclude
|
|
// during discovery. For Cloud Storage bucket assets, these are interpreted
|
|
// as glob patterns used to match object names. For BigQuery dataset assets,
|
|
// these are interpreted as patterns to match table names.
|
|
repeated string exclude_patterns = 3
|
|
[(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Configuration for CSV data.
|
|
CsvOptions csv_options = 4 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Configuration for Json data.
|
|
JsonOptions json_options = 5 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Determines when discovery is triggered.
|
|
oneof trigger {
|
|
// Optional. Cron schedule (https://en.wikipedia.org/wiki/Cron) for
|
|
// running discovery periodically. Successive discovery runs must be
|
|
// scheduled at least 60 minutes apart. The default value is to run
|
|
// discovery every 60 minutes.
|
|
//
|
|
// To explicitly set a timezone to the cron tab, apply a prefix in the
|
|
// cron tab: "CRON_TZ=${IANA_TIME_ZONE}" or TZ=${IANA_TIME_ZONE}".
|
|
// The ${IANA_TIME_ZONE} may only be a valid string from IANA time zone
|
|
// database. For example, `CRON_TZ=America/New_York 1 * * * *`, or
|
|
// `TZ=America/New_York 1 * * * *`.
|
|
string schedule = 10 [(google.api.field_behavior) = OPTIONAL];
|
|
}
|
|
}
|
|
|
|
// Output only. The relative resource name of the zone, of the form:
|
|
// `projects/{project_number}/locations/{location_id}/lakes/{lake_id}/zones/{zone_id}`.
|
|
string name = 1 [
|
|
(google.api.field_behavior) = OUTPUT_ONLY,
|
|
(google.api.resource_reference) = { type: "dataplex.googleapis.com/Zone" }
|
|
];
|
|
|
|
// Optional. User friendly display name.
|
|
string display_name = 2 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Output only. System generated globally unique ID for the zone. This ID will
|
|
// be different if the zone is deleted and re-created with the same name.
|
|
string uid = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
|
|
|
|
// Output only. The time when the zone was created.
|
|
google.protobuf.Timestamp create_time = 4
|
|
[(google.api.field_behavior) = OUTPUT_ONLY];
|
|
|
|
// Output only. The time when the zone was last updated.
|
|
google.protobuf.Timestamp update_time = 5
|
|
[(google.api.field_behavior) = OUTPUT_ONLY];
|
|
|
|
// Optional. User defined labels for the zone.
|
|
map<string, string> labels = 6 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Description of the zone.
|
|
string description = 7 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Output only. Current state of the zone.
|
|
State state = 8 [(google.api.field_behavior) = OUTPUT_ONLY];
|
|
|
|
// Required. Immutable. The type of the zone.
|
|
Type type = 9 [
|
|
(google.api.field_behavior) = REQUIRED,
|
|
(google.api.field_behavior) = IMMUTABLE
|
|
];
|
|
|
|
// Optional. Specification of the discovery feature applied to data in this
|
|
// zone.
|
|
DiscoverySpec discovery_spec = 103 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Required. Specification of the resources that are referenced by the assets
|
|
// within this zone.
|
|
ResourceSpec resource_spec = 104 [(google.api.field_behavior) = REQUIRED];
|
|
|
|
// Output only. Aggregated status of the underlying assets of the zone.
|
|
AssetStatus asset_status = 105 [(google.api.field_behavior) = OUTPUT_ONLY];
|
|
}
|
|
|
|
// Action represents an issue requiring administrator action for resolution.
|
|
message Action {
|
|
option (google.api.resource) = {
|
|
type: "dataplex.googleapis.com/Action"
|
|
pattern: "projects/{project}/locations/{location}/lakes/{lake}/actions/{action}"
|
|
pattern: "projects/{project}/locations/{location}/lakes/{lake}/zones/{zone}/actions/{action}"
|
|
pattern: "projects/{project}/locations/{location}/lakes/{lake}/zones/{zone}/assets/{asset}/actions/{action}"
|
|
};
|
|
|
|
// The category of issues.
|
|
enum Category {
|
|
// Unspecified category.
|
|
CATEGORY_UNSPECIFIED = 0;
|
|
|
|
// Resource management related issues.
|
|
RESOURCE_MANAGEMENT = 1;
|
|
|
|
// Security policy related issues.
|
|
SECURITY_POLICY = 2;
|
|
|
|
// Data and discovery related issues.
|
|
DATA_DISCOVERY = 3;
|
|
}
|
|
|
|
// Action details for resource references in assets that cannot be located.
|
|
message MissingResource {}
|
|
|
|
// Action details for unauthorized resource issues raised to indicate that the
|
|
// service account associated with the lake instance is not authorized to
|
|
// access or manage the resource associated with an asset.
|
|
message UnauthorizedResource {}
|
|
|
|
// Failed to apply security policy to the managed resource(s) under a
|
|
// lake, zone or an asset. For a lake or zone resource, one or more underlying
|
|
// assets has a failure applying security policy to the associated managed
|
|
// resource.
|
|
message FailedSecurityPolicyApply {
|
|
// Resource name of one of the assets with failing security policy
|
|
// application. Populated for a lake or zone resource only.
|
|
string asset = 1;
|
|
}
|
|
|
|
// Action details for invalid or unsupported data files detected by discovery.
|
|
message InvalidDataFormat {
|
|
// The list of data locations sampled and used for format/schema
|
|
// inference.
|
|
repeated string sampled_data_locations = 1;
|
|
|
|
// The expected data format of the entity.
|
|
string expected_format = 2;
|
|
|
|
// The new unexpected data format within the entity.
|
|
string new_format = 3;
|
|
}
|
|
|
|
// Action details for incompatible schemas detected by discovery.
|
|
message IncompatibleDataSchema {
|
|
// Whether the action relates to a schema that is incompatible or modified.
|
|
enum SchemaChange {
|
|
// Schema change unspecified.
|
|
SCHEMA_CHANGE_UNSPECIFIED = 0;
|
|
|
|
// Newly discovered schema is incompatible with existing schema.
|
|
INCOMPATIBLE = 1;
|
|
|
|
// Newly discovered schema has changed from existing schema for data in a
|
|
// curated zone.
|
|
MODIFIED = 2;
|
|
}
|
|
|
|
// The name of the table containing invalid data.
|
|
string table = 1;
|
|
|
|
// The existing and expected schema of the table. The schema is provided as
|
|
// a JSON formatted structure listing columns and data types.
|
|
string existing_schema = 2;
|
|
|
|
// The new and incompatible schema within the table. The schema is provided
|
|
// as a JSON formatted structured listing columns and data types.
|
|
string new_schema = 3;
|
|
|
|
// The list of data locations sampled and used for format/schema
|
|
// inference.
|
|
repeated string sampled_data_locations = 4;
|
|
|
|
// Whether the action relates to a schema that is incompatible or modified.
|
|
SchemaChange schema_change = 5;
|
|
}
|
|
|
|
// Action details for invalid or unsupported partitions detected by discovery.
|
|
message InvalidDataPartition {
|
|
// The expected partition structure.
|
|
enum PartitionStructure {
|
|
// PartitionStructure unspecified.
|
|
PARTITION_STRUCTURE_UNSPECIFIED = 0;
|
|
|
|
// Consistent hive-style partition definition (both raw and curated zone).
|
|
CONSISTENT_KEYS = 1;
|
|
|
|
// Hive style partition definition (curated zone only).
|
|
HIVE_STYLE_KEYS = 2;
|
|
}
|
|
|
|
// The issue type of InvalidDataPartition.
|
|
PartitionStructure expected_structure = 1;
|
|
}
|
|
|
|
// Action details for absence of data detected by discovery.
|
|
message MissingData {}
|
|
|
|
// Action details for invalid data arrangement.
|
|
message InvalidDataOrganization {}
|
|
|
|
// The category of issue associated with the action.
|
|
Category category = 1;
|
|
|
|
// Detailed description of the issue requiring action.
|
|
string issue = 2;
|
|
|
|
// The time that the issue was detected.
|
|
google.protobuf.Timestamp detect_time = 4;
|
|
|
|
// Output only. The relative resource name of the action, of the form:
|
|
// `projects/{project}/locations/{location}/lakes/{lake}/actions/{action}`
|
|
// `projects/{project}/locations/{location}/lakes/{lake}/zones/{zone}/actions/{action}`
|
|
// `projects/{project}/locations/{location}/lakes/{lake}/zones/{zone}/assets/{asset}/actions/{action}`.
|
|
string name = 5 [
|
|
(google.api.field_behavior) = OUTPUT_ONLY,
|
|
(google.api.resource_reference) = { type: "dataplex.googleapis.com/Action" }
|
|
];
|
|
|
|
// Output only. The relative resource name of the lake, of the form:
|
|
// `projects/{project_number}/locations/{location_id}/lakes/{lake_id}`.
|
|
string lake = 6 [
|
|
(google.api.field_behavior) = OUTPUT_ONLY,
|
|
(google.api.resource_reference) = { type: "dataplex.googleapis.com/Lake" }
|
|
];
|
|
|
|
// Output only. The relative resource name of the zone, of the form:
|
|
// `projects/{project_number}/locations/{location_id}/lakes/{lake_id}/zones/{zone_id}`.
|
|
string zone = 7 [
|
|
(google.api.field_behavior) = OUTPUT_ONLY,
|
|
(google.api.resource_reference) = { type: "dataplex.googleapis.com/Zone" }
|
|
];
|
|
|
|
// Output only. The relative resource name of the asset, of the form:
|
|
// `projects/{project_number}/locations/{location_id}/lakes/{lake_id}/zones/{zone_id}/assets/{asset_id}`.
|
|
string asset = 8 [
|
|
(google.api.field_behavior) = OUTPUT_ONLY,
|
|
(google.api.resource_reference) = { type: "dataplex.googleapis.com/Asset" }
|
|
];
|
|
|
|
// The list of data locations associated with this action. Cloud Storage
|
|
// locations are represented as URI paths(E.g.
|
|
// `gs://bucket/table1/year=2020/month=Jan/`). BigQuery locations refer to
|
|
// resource names(E.g.
|
|
// `bigquery.googleapis.com/projects/project-id/datasets/dataset-id`).
|
|
repeated string data_locations = 9;
|
|
|
|
// Additional details about the action based on the action category.
|
|
oneof details {
|
|
// Details for issues related to invalid or unsupported data formats.
|
|
InvalidDataFormat invalid_data_format = 10;
|
|
|
|
// Details for issues related to incompatible schemas detected within data.
|
|
IncompatibleDataSchema incompatible_data_schema = 11;
|
|
|
|
// Details for issues related to invalid or unsupported data partition
|
|
// structure.
|
|
InvalidDataPartition invalid_data_partition = 12;
|
|
|
|
// Details for issues related to absence of data within managed resources.
|
|
MissingData missing_data = 13;
|
|
|
|
// Details for issues related to absence of a managed resource.
|
|
MissingResource missing_resource = 14;
|
|
|
|
// Details for issues related to lack of permissions to access data
|
|
// resources.
|
|
UnauthorizedResource unauthorized_resource = 15;
|
|
|
|
// Details for issues related to applying security policy.
|
|
FailedSecurityPolicyApply failed_security_policy_apply = 21;
|
|
|
|
// Details for issues related to invalid data arrangement.
|
|
InvalidDataOrganization invalid_data_organization = 22;
|
|
}
|
|
}
|
|
|
|
// An asset represents a cloud resource that is being managed within a lake as a
|
|
// member of a zone.
|
|
message Asset {
|
|
option (google.api.resource) = {
|
|
type: "dataplex.googleapis.com/Asset"
|
|
pattern: "projects/{project}/locations/{location}/lakes/{lake}/zones/{zone}/assets/{asset}"
|
|
};
|
|
|
|
// Security policy status of the asset. Data security policy, i.e., readers,
|
|
// writers & owners, should be specified in the lake/zone/asset IAM policy.
|
|
message SecurityStatus {
|
|
// The state of the security policy.
|
|
enum State {
|
|
// State unspecified.
|
|
STATE_UNSPECIFIED = 0;
|
|
|
|
// Security policy has been successfully applied to the attached resource.
|
|
READY = 1;
|
|
|
|
// Security policy is in the process of being applied to the attached
|
|
// resource.
|
|
APPLYING = 2;
|
|
|
|
// Security policy could not be applied to the attached resource due to
|
|
// errors.
|
|
ERROR = 3;
|
|
}
|
|
|
|
// The current state of the security policy applied to the attached
|
|
// resource.
|
|
State state = 1;
|
|
|
|
// Additional information about the current state.
|
|
string message = 2;
|
|
|
|
// Last update time of the status.
|
|
google.protobuf.Timestamp update_time = 3;
|
|
}
|
|
|
|
// Settings to manage the metadata discovery and publishing for an asset.
|
|
message DiscoverySpec {
|
|
// Describe CSV and similar semi-structured data formats.
|
|
message CsvOptions {
|
|
// Optional. The number of rows to interpret as header rows that should be
|
|
// skipped when reading data rows.
|
|
int32 header_rows = 1 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. The delimiter being used to separate values. This defaults to
|
|
// ','.
|
|
string delimiter = 2 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. The character encoding of the data. The default is UTF-8.
|
|
string encoding = 3 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Whether to disable the inference of data type for CSV data.
|
|
// If true, all columns will be registered as strings.
|
|
bool disable_type_inference = 4 [(google.api.field_behavior) = OPTIONAL];
|
|
}
|
|
|
|
// Describe JSON data format.
|
|
message JsonOptions {
|
|
// Optional. The character encoding of the data. The default is UTF-8.
|
|
string encoding = 1 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Whether to disable the inference of data type for Json data.
|
|
// If true, all columns will be registered as their primitive types
|
|
// (strings, number or boolean).
|
|
bool disable_type_inference = 2 [(google.api.field_behavior) = OPTIONAL];
|
|
}
|
|
|
|
// Optional. Whether discovery is enabled.
|
|
bool enabled = 1 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. The list of patterns to apply for selecting data to include
|
|
// during discovery if only a subset of the data should considered. For
|
|
// Cloud Storage bucket assets, these are interpreted as glob patterns used
|
|
// to match object names. For BigQuery dataset assets, these are interpreted
|
|
// as patterns to match table names.
|
|
repeated string include_patterns = 2
|
|
[(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. The list of patterns to apply for selecting data to exclude
|
|
// during discovery. For Cloud Storage bucket assets, these are interpreted
|
|
// as glob patterns used to match object names. For BigQuery dataset assets,
|
|
// these are interpreted as patterns to match table names.
|
|
repeated string exclude_patterns = 3
|
|
[(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Configuration for CSV data.
|
|
CsvOptions csv_options = 4 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Configuration for Json data.
|
|
JsonOptions json_options = 5 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Determines when discovery is triggered.
|
|
oneof trigger {
|
|
// Optional. Cron schedule (https://en.wikipedia.org/wiki/Cron) for
|
|
// running discovery periodically. Successive discovery runs must be
|
|
// scheduled at least 60 minutes apart. The default value is to run
|
|
// discovery every 60 minutes.
|
|
//
|
|
// To explicitly set a timezone to the cron tab, apply a prefix in the
|
|
// cron tab: "CRON_TZ=${IANA_TIME_ZONE}" or TZ=${IANA_TIME_ZONE}".
|
|
// The ${IANA_TIME_ZONE} may only be a valid string from IANA time zone
|
|
// database. For example, `CRON_TZ=America/New_York 1 * * * *`, or
|
|
// `TZ=America/New_York 1 * * * *`.
|
|
string schedule = 10 [(google.api.field_behavior) = OPTIONAL];
|
|
}
|
|
}
|
|
|
|
// Identifies the cloud resource that is referenced by this asset.
|
|
message ResourceSpec {
|
|
// Type of resource.
|
|
enum Type {
|
|
// Type not specified.
|
|
TYPE_UNSPECIFIED = 0;
|
|
|
|
// Cloud Storage bucket.
|
|
STORAGE_BUCKET = 1;
|
|
|
|
// BigQuery dataset.
|
|
BIGQUERY_DATASET = 2;
|
|
}
|
|
|
|
// Access Mode determines how data stored within the resource is read. This
|
|
// is only applicable to storage bucket assets.
|
|
enum AccessMode {
|
|
// Access mode unspecified.
|
|
ACCESS_MODE_UNSPECIFIED = 0;
|
|
|
|
// Default. Data is accessed directly using storage APIs.
|
|
DIRECT = 1;
|
|
|
|
// Data is accessed through a managed interface using BigQuery APIs.
|
|
MANAGED = 2;
|
|
}
|
|
|
|
// Immutable. Relative name of the cloud resource that contains the data
|
|
// that is being managed within a lake. For example:
|
|
// `projects/{project_number}/buckets/{bucket_id}`
|
|
// `projects/{project_number}/datasets/{dataset_id}`
|
|
string name = 1 [(google.api.field_behavior) = IMMUTABLE];
|
|
|
|
// Required. Immutable. Type of resource.
|
|
Type type = 2 [
|
|
(google.api.field_behavior) = REQUIRED,
|
|
(google.api.field_behavior) = IMMUTABLE
|
|
];
|
|
|
|
// Optional. Determines how read permissions are handled for each asset and
|
|
// their associated tables. Only available to storage buckets assets.
|
|
AccessMode read_access_mode = 5 [(google.api.field_behavior) = OPTIONAL];
|
|
}
|
|
|
|
// Status of the resource referenced by an asset.
|
|
message ResourceStatus {
|
|
// The state of a resource.
|
|
enum State {
|
|
// State unspecified.
|
|
STATE_UNSPECIFIED = 0;
|
|
|
|
// Resource does not have any errors.
|
|
READY = 1;
|
|
|
|
// Resource has errors.
|
|
ERROR = 2;
|
|
}
|
|
|
|
// The current state of the managed resource.
|
|
State state = 1;
|
|
|
|
// Additional information about the current state.
|
|
string message = 2;
|
|
|
|
// Last update time of the status.
|
|
google.protobuf.Timestamp update_time = 3;
|
|
|
|
// Output only. Service account associated with the BigQuery Connection.
|
|
string managed_access_identity = 4
|
|
[(google.api.field_behavior) = OUTPUT_ONLY];
|
|
}
|
|
|
|
// Status of discovery for an asset.
|
|
message DiscoveryStatus {
|
|
// Current state of discovery.
|
|
enum State {
|
|
// State is unspecified.
|
|
STATE_UNSPECIFIED = 0;
|
|
|
|
// Discovery for the asset is scheduled.
|
|
SCHEDULED = 1;
|
|
|
|
// Discovery for the asset is running.
|
|
IN_PROGRESS = 2;
|
|
|
|
// Discovery for the asset is currently paused (e.g. due to a lack
|
|
// of available resources). It will be automatically resumed.
|
|
PAUSED = 3;
|
|
|
|
// Discovery for the asset is disabled.
|
|
DISABLED = 5;
|
|
}
|
|
|
|
// The aggregated data statistics for the asset reported by discovery.
|
|
message Stats {
|
|
// The count of data items within the referenced resource.
|
|
int64 data_items = 1;
|
|
|
|
// The number of stored data bytes within the referenced resource.
|
|
int64 data_size = 2;
|
|
|
|
// The count of table entities within the referenced resource.
|
|
int64 tables = 3;
|
|
|
|
// The count of fileset entities within the referenced resource.
|
|
int64 filesets = 4;
|
|
}
|
|
|
|
// The current status of the discovery feature.
|
|
State state = 1;
|
|
|
|
// Additional information about the current state.
|
|
string message = 2;
|
|
|
|
// Last update time of the status.
|
|
google.protobuf.Timestamp update_time = 3;
|
|
|
|
// The start time of the last discovery run.
|
|
google.protobuf.Timestamp last_run_time = 4;
|
|
|
|
// Data Stats of the asset reported by discovery.
|
|
Stats stats = 6;
|
|
|
|
// The duration of the last discovery run.
|
|
google.protobuf.Duration last_run_duration = 7;
|
|
}
|
|
|
|
// Output only. The relative resource name of the asset, of the form:
|
|
// `projects/{project_number}/locations/{location_id}/lakes/{lake_id}/zones/{zone_id}/assets/{asset_id}`.
|
|
string name = 1 [
|
|
(google.api.field_behavior) = OUTPUT_ONLY,
|
|
(google.api.resource_reference) = { type: "dataplex.googleapis.com/Asset" }
|
|
];
|
|
|
|
// Optional. User friendly display name.
|
|
string display_name = 2 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Output only. System generated globally unique ID for the asset. This ID
|
|
// will be different if the asset is deleted and re-created with the same
|
|
// name.
|
|
string uid = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
|
|
|
|
// Output only. The time when the asset was created.
|
|
google.protobuf.Timestamp create_time = 4
|
|
[(google.api.field_behavior) = OUTPUT_ONLY];
|
|
|
|
// Output only. The time when the asset was last updated.
|
|
google.protobuf.Timestamp update_time = 5
|
|
[(google.api.field_behavior) = OUTPUT_ONLY];
|
|
|
|
// Optional. User defined labels for the asset.
|
|
map<string, string> labels = 6 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Optional. Description of the asset.
|
|
string description = 7 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Output only. Current state of the asset.
|
|
State state = 8 [(google.api.field_behavior) = OUTPUT_ONLY];
|
|
|
|
// Required. Specification of the resource that is referenced by this asset.
|
|
ResourceSpec resource_spec = 100 [(google.api.field_behavior) = REQUIRED];
|
|
|
|
// Output only. Status of the resource referenced by this asset.
|
|
ResourceStatus resource_status = 101
|
|
[(google.api.field_behavior) = OUTPUT_ONLY];
|
|
|
|
// Output only. Status of the security policy applied to resource referenced
|
|
// by this asset.
|
|
SecurityStatus security_status = 103
|
|
[(google.api.field_behavior) = OUTPUT_ONLY];
|
|
|
|
// Optional. Specification of the discovery feature applied to data referenced
|
|
// by this asset. When this spec is left unset, the asset will use the spec
|
|
// set on the parent zone.
|
|
DiscoverySpec discovery_spec = 106 [(google.api.field_behavior) = OPTIONAL];
|
|
|
|
// Output only. Status of the discovery feature applied to data referenced by
|
|
// this asset.
|
|
DiscoveryStatus discovery_status = 107
|
|
[(google.api.field_behavior) = OUTPUT_ONLY];
|
|
}
|
|
|
|
// State of a resource.
|
|
enum State {
|
|
// State is not specified.
|
|
STATE_UNSPECIFIED = 0;
|
|
|
|
// Resource is active, i.e., ready to use.
|
|
ACTIVE = 1;
|
|
|
|
// Resource is under creation.
|
|
CREATING = 2;
|
|
|
|
// Resource is under deletion.
|
|
DELETING = 3;
|
|
|
|
// Resource is active but has unresolved actions.
|
|
ACTION_REQUIRED = 4;
|
|
}
|