From dbc83bd75946f2dbbca5fbb2be13d6afbc635b83 Mon Sep 17 00:00:00 2001 From: Google APIs Date: Tue, 20 Sep 2022 09:49:42 -0700 Subject: [PATCH] feat: add timestamp_outside_retention_rows_count to ImportFeatureValuesResponse and ImportFeatureValuesOperationMetadata in aiplatform v1beta1 featurestore_service.proto feat: add RemoveContextChildren rpc to aiplatform v1beta1 metadata_service.proto feat: add order_by to ListArtifactsRequest, ListContextsRequest, and ListExecutionsRequest in aiplatform v1beta1 metadata_service.proto feat: add InputArtifact to RuntimeConfig in aiplatform v1beta1 pipeline_job.proto feat: add read_mask to ListPipelineJobsRequest in aiplatform v1beta1 pipeline_service.proto feat: add TransferLearningConfig in aiplatform v1beta1 study.proto PiperOrigin-RevId: 475580307 --- google/cloud/aiplatform/v1beta1/BUILD.bazel | 2 + .../v1beta1/batch_prediction_job.proto | 2 +- .../cloud/aiplatform/v1beta1/custom_job.proto | 1 - .../v1beta1/featurestore_service.proto | 55 +++++++++++++---- .../aiplatform/v1beta1/job_service.proto | 25 ++++++++ .../aiplatform/v1beta1/metadata_service.proto | 60 +++++++++++++++++++ .../aiplatform/v1beta1/model_monitoring.proto | 13 ++-- .../aiplatform/v1beta1/pipeline_job.proto | 16 +++++ .../aiplatform/v1beta1/pipeline_service.proto | 6 ++ google/cloud/aiplatform/v1beta1/study.proto | 17 ++++++ 10 files changed, 174 insertions(+), 23 deletions(-) diff --git a/google/cloud/aiplatform/v1beta1/BUILD.bazel b/google/cloud/aiplatform/v1beta1/BUILD.bazel index 5fa782469e..ae11c9d6e6 100644 --- a/google/cloud/aiplatform/v1beta1/BUILD.bazel +++ b/google/cloud/aiplatform/v1beta1/BUILD.bazel @@ -106,6 +106,7 @@ proto_library( "//google/api:resource_proto", "//google/longrunning:operations_proto", "//google/rpc:status_proto", + "//google/type:interval_proto", "//google/type:money_proto", "@com_google_protobuf//:duration_proto", "@com_google_protobuf//:empty_proto", @@ -236,6 +237,7 @@ go_proto_library( "//google/api:httpbody_go_proto", "//google/longrunning:longrunning_go_proto", "//google/rpc:status_go_proto", + "//google/type:interval_go_proto", "//google/type:money_go_proto", ], ) diff --git a/google/cloud/aiplatform/v1beta1/batch_prediction_job.proto b/google/cloud/aiplatform/v1beta1/batch_prediction_job.proto index 5141dfa274..63fb0d9d0c 100644 --- a/google/cloud/aiplatform/v1beta1/batch_prediction_job.proto +++ b/google/cloud/aiplatform/v1beta1/batch_prediction_job.proto @@ -156,7 +156,7 @@ message BatchPredictionJob { // Required. The user-defined name of this BatchPredictionJob. string display_name = 2 [(google.api.field_behavior) = REQUIRED]; - // The name of the Model resoure that produces the predictions via this job, + // The name of the Model resource that produces the predictions via this job, // must share the same ancestor Location. // Starting this job has no impact on any existing deployments of the Model // and their resources. diff --git a/google/cloud/aiplatform/v1beta1/custom_job.proto b/google/cloud/aiplatform/v1beta1/custom_job.proto index ad66542d35..24d384ff60 100644 --- a/google/cloud/aiplatform/v1beta1/custom_job.proto +++ b/google/cloud/aiplatform/v1beta1/custom_job.proto @@ -106,7 +106,6 @@ message CustomJob { } // Represents the spec of a CustomJob. -// Next Id: 15 message CustomJobSpec { // Required. The spec of the worker pools including machine type and Docker image. // All worker pools except the first one are optional and can be skipped by diff --git a/google/cloud/aiplatform/v1beta1/featurestore_service.proto b/google/cloud/aiplatform/v1beta1/featurestore_service.proto index 6acace9e7d..56df4670f7 100644 --- a/google/cloud/aiplatform/v1beta1/featurestore_service.proto +++ b/google/cloud/aiplatform/v1beta1/featurestore_service.proto @@ -29,6 +29,7 @@ import "google/cloud/aiplatform/v1beta1/operation.proto"; import "google/longrunning/operations.proto"; import "google/protobuf/field_mask.proto"; import "google/protobuf/timestamp.proto"; +import "google/type/interval.proto"; option csharp_namespace = "Google.Cloud.AIPlatform.V1Beta1"; option go_package = "google.golang.org/genproto/googleapis/cloud/aiplatform/v1beta1;aiplatform"; @@ -290,8 +291,7 @@ service FeaturestoreService { // returned from reads and exports may be inconsistent. If consistency is // required, the caller must retry the same delete request again and wait till // the new operation returned is marked as successfully done. - rpc DeleteFeatureValues(DeleteFeatureValuesRequest) - returns (google.longrunning.Operation) { + rpc DeleteFeatureValues(DeleteFeatureValuesRequest) returns (google.longrunning.Operation) { option (google.api.http) = { post: "/v1beta1/{entity_type=projects/*/locations/*/featurestores/*/entityTypes/*}:deleteFeatureValues" body: "*" @@ -548,6 +548,10 @@ message ImportFeatureValuesResponse { // * Having a null timestamp. // * Not being parsable (applicable for CSV sources). int64 invalid_row_count = 6; + + // The number rows that weren't ingested due to having feature timestamps + // outside the retention boundary. + int64 timestamp_outside_retention_rows_count = 4; } // Request message for [FeaturestoreService.BatchReadFeatureValues][google.cloud.aiplatform.v1beta1.FeaturestoreService.BatchReadFeatureValues]. @@ -1216,6 +1220,10 @@ message ImportFeatureValuesOperationMetadata { // * Having a null timestamp. // * Not being parsable (applicable for CSV sources). int64 invalid_row_count = 6; + + // The number rows that weren't ingested due to having timestamps outside the + // retention boundary. + int64 timestamp_outside_retention_rows_count = 7; } // Details of operations that exports Features values. @@ -1254,27 +1262,47 @@ message BatchCreateFeaturesOperationMetadata { GenericOperationMetadata generic_metadata = 1; } -// Request message for -// [FeaturestoreService.DeleteFeatureValues][google.cloud.aiplatform.v1beta1.FeaturestoreService.DeleteFeatureValues]. +// Request message for [FeaturestoreService.DeleteFeatureValues][google.cloud.aiplatform.v1beta1.FeaturestoreService.DeleteFeatureValues]. message DeleteFeatureValuesRequest { // Message to select entity. // If an entity id is selected, all the feature values corresponding to the // entity id will be deleted, including the entityId. message SelectEntity { - // Required. Selectors choosing feature values of which entity id to be - // deleted from the EntityType. - EntityIdSelector entity_id_selector = 1 - [(google.api.field_behavior) = REQUIRED]; + // Required. Selectors choosing feature values of which entity id to be deleted from + // the EntityType. + EntityIdSelector entity_id_selector = 1 [(google.api.field_behavior) = REQUIRED]; + } + + // Message to select time range and feature. + // Values of the selected feature generated within an inclusive time range + // will be deleted. + message SelectTimeRangeAndFeature { + // Required. Select feature generated within a half-inclusive time range. + // The time range is lower inclusive and upper exclusive. + google.type.Interval time_range = 1 [(google.api.field_behavior) = REQUIRED]; + + // Required. Selectors choosing which feature values to be deleted from the + // EntityType. + FeatureSelector feature_selector = 2 [(google.api.field_behavior) = REQUIRED]; + + // If set, data will not be deleted from online storage. + // When time range is older than the data in online storage, setting this to + // be true will make the deletion have no impact on online serving. + bool skip_online_storage_delete = 3; } // Defines options to select feature values to be deleted. oneof DeleteOption { // Select feature values to be deleted by specifying entities. SelectEntity select_entity = 2; + + // Select feature values to be deleted by specifying time range and + // features. + SelectTimeRangeAndFeature select_time_range_and_feature = 3; } - // Required. The resource name of the EntityType grouping the Features for - // which values are being deleted from. Format: + // Required. The resource name of the EntityType grouping the Features for which values + // are being deleted from. Format: // `projects/{project}/locations/{location}/featurestores/{featurestore}/entityTypes/{entityType}` string entity_type = 1 [ (google.api.field_behavior) = REQUIRED, @@ -1284,9 +1312,10 @@ message DeleteFeatureValuesRequest { ]; } -// Response message for -// [FeaturestoreService.DeleteFeatureValues][google.cloud.aiplatform.v1beta1.FeaturestoreService.DeleteFeatureValues]. -message DeleteFeatureValuesResponse {} +// Response message for [FeaturestoreService.DeleteFeatureValues][google.cloud.aiplatform.v1beta1.FeaturestoreService.DeleteFeatureValues]. +message DeleteFeatureValuesResponse { + +} // Selector for entityId. Getting ids from the given source. message EntityIdSelector { diff --git a/google/cloud/aiplatform/v1beta1/job_service.proto b/google/cloud/aiplatform/v1beta1/job_service.proto index 48cf08aa79..68492a64c1 100644 --- a/google/cloud/aiplatform/v1beta1/job_service.proto +++ b/google/cloud/aiplatform/v1beta1/job_service.proto @@ -388,6 +388,9 @@ message ListCustomJobsRequest { // * `state` supports `=`, `!=` comparisons. // * `create_time` supports `=`, `!=`,`<`, `<=`,`>`, `>=` comparisons. // `create_time` must be in RFC 3339 format. + // * `labels` supports general map functions that is: + // `labels.key=value` - key:value equality + // `labels.key:* - key existence // // Some examples of using the filter are: // @@ -395,6 +398,8 @@ message ListCustomJobsRequest { // * `state!="JOB_STATE_FAILED" OR display_name="my_job"` // * `NOT display_name="my_job"` // * `create_time>"2021-05-18T00:00:00Z"` + // * `labels.keyA=valueA` + // * `labels.keyB:*` string filter = 2; // The standard list page size. @@ -493,6 +498,9 @@ message ListDataLabelingJobsRequest { // * `state` supports `=`, `!=` comparisons. // * `create_time` supports `=`, `!=`,`<`, `<=`,`>`, `>=` comparisons. // `create_time` must be in RFC 3339 format. + // * `labels` supports general map functions that is: + // `labels.key=value` - key:value equality + // `labels.key:* - key existence // // Some examples of using the filter are: // @@ -500,6 +508,8 @@ message ListDataLabelingJobsRequest { // * `state!="JOB_STATE_FAILED" OR display_name="my_job"` // * `NOT display_name="my_job"` // * `create_time>"2021-05-18T00:00:00Z"` + // * `labels.keyA=valueA` + // * `labels.keyB:*` string filter = 2; // The standard list page size. @@ -603,6 +613,9 @@ message ListHyperparameterTuningJobsRequest { // * `state` supports `=`, `!=` comparisons. // * `create_time` supports `=`, `!=`,`<`, `<=`,`>`, `>=` comparisons. // `create_time` must be in RFC 3339 format. + // * `labels` supports general map functions that is: + // `labels.key=value` - key:value equality + // `labels.key:* - key existence // // Some examples of using the filter are: // @@ -610,6 +623,8 @@ message ListHyperparameterTuningJobsRequest { // * `state!="JOB_STATE_FAILED" OR display_name="my_job"` // * `NOT display_name="my_job"` // * `create_time>"2021-05-18T00:00:00Z"` + // * `labels.keyA=valueA` + // * `labels.keyB:*` string filter = 2; // The standard list page size. @@ -711,6 +726,9 @@ message ListBatchPredictionJobsRequest { // * `state` supports `=`, `!=` comparisons. // * `create_time` supports `=`, `!=`,`<`, `<=`,`>`, `>=` comparisons. // `create_time` must be in RFC 3339 format. + // * `labels` supports general map functions that is: + // `labels.key=value` - key:value equality + // `labels.key:* - key existence // // Some examples of using the filter are: // @@ -718,6 +736,8 @@ message ListBatchPredictionJobsRequest { // * `state!="JOB_STATE_FAILED" OR display_name="my_job"` // * `NOT display_name="my_job"` // * `create_time>"2021-05-18T00:00:00Z"` + // * `labels.keyA=valueA` + // * `labels.keyB:*` string filter = 2; // The standard list page size. @@ -891,6 +911,9 @@ message ListModelDeploymentMonitoringJobsRequest { // * `state` supports `=`, `!=` comparisons. // * `create_time` supports `=`, `!=`,`<`, `<=`,`>`, `>=` comparisons. // `create_time` must be in RFC 3339 format. + // * `labels` supports general map functions that is: + // `labels.key=value` - key:value equality + // `labels.key:* - key existence // // Some examples of using the filter are: // @@ -898,6 +921,8 @@ message ListModelDeploymentMonitoringJobsRequest { // * `state!="JOB_STATE_FAILED" OR display_name="my_job"` // * `NOT display_name="my_job"` // * `create_time>"2021-05-18T00:00:00Z"` + // * `labels.keyA=valueA` + // * `labels.keyB:*` string filter = 2; // The standard list page size. diff --git a/google/cloud/aiplatform/v1beta1/metadata_service.proto b/google/cloud/aiplatform/v1beta1/metadata_service.proto index afcf68d0be..abdcb600c2 100644 --- a/google/cloud/aiplatform/v1beta1/metadata_service.proto +++ b/google/cloud/aiplatform/v1beta1/metadata_service.proto @@ -228,6 +228,17 @@ service MetadataService { option (google.api.method_signature) = "context,child_contexts"; } + // Remove a set of children contexts from a parent Context. If any of the + // child Contexts were NOT added to the parent Context, they are + // simply skipped. + rpc RemoveContextChildren(RemoveContextChildrenRequest) returns (RemoveContextChildrenResponse) { + option (google.api.http) = { + post: "/v1beta1/{context=projects/*/locations/*/metadataStores/*/contexts/*}:removeContextChildren" + body: "*" + }; + option (google.api.method_signature) = "context,child_contexts"; + } + // Retrieves Artifacts and Executions within the specified Context, connected // by Event edges and returned as a LineageSubgraph. rpc QueryContextLineageSubgraph(QueryContextLineageSubgraphRequest) returns (LineageSubgraph) { @@ -549,6 +560,14 @@ message ListArtifactsRequest { // // For example: `display_name = "test" AND metadata.field1.bool_value = true`. string filter = 4; + + // How the list of messages is ordered. Specify the values to order by and an + // ordering operation. The default sorting order is ascending. To specify + // descending order for a field, users append a " desc" suffix; for example: + // "foo desc, bar". + // Subfields are specified with a `.` character, such as foo.bar. + // see https://google.aip.dev/132#ordering for more details. + string order_by = 5; } // Response message for [MetadataService.ListArtifacts][google.cloud.aiplatform.v1beta1.MetadataService.ListArtifacts]. @@ -735,6 +754,14 @@ message ListContextsRequest { // // For example: `display_name = "test" AND metadata.field1.bool_value = true`. string filter = 4; + + // How the list of messages is ordered. Specify the values to order by and an + // ordering operation. The default sorting order is ascending. To specify + // descending order for a field, users append a " desc" suffix; for example: + // "foo desc, bar". + // Subfields are specified with a `.` character, such as foo.bar. + // see https://google.aip.dev/132#ordering for more details. + string order_by = 5; } // Response message for [MetadataService.ListContexts][google.cloud.aiplatform.v1beta1.MetadataService.ListContexts]. @@ -890,6 +917,31 @@ message AddContextChildrenResponse { } +// Request message for +// [MetadataService.DeleteContextChildrenRequest][]. +message RemoveContextChildrenRequest { + // Required. The resource name of the parent Context. + // + // Format: + // `projects/{project}/locations/{location}/metadataStores/{metadatastore}/contexts/{context}` + string context = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "aiplatform.googleapis.com/Context" + } + ]; + + // The resource names of the child Contexts. + repeated string child_contexts = 2 [(google.api.resource_reference) = { + type: "aiplatform.googleapis.com/Context" + }]; +} + +// Response message for [MetadataService.RemoveContextChildren][google.cloud.aiplatform.v1beta1.MetadataService.RemoveContextChildren]. +message RemoveContextChildrenResponse { + +} + // Request message for [MetadataService.QueryContextLineageSubgraph][google.cloud.aiplatform.v1beta1.MetadataService.QueryContextLineageSubgraph]. message QueryContextLineageSubgraphRequest { // Required. The resource name of the Context whose Artifacts and Executions @@ -999,6 +1051,14 @@ message ListExecutionsRequest { // logical operators (`AND` & `OR`). // For example: `display_name = "test" AND metadata.field1.bool_value = true`. string filter = 4; + + // How the list of messages is ordered. Specify the values to order by and an + // ordering operation. The default sorting order is ascending. To specify + // descending order for a field, users append a " desc" suffix; for example: + // "foo desc, bar". + // Subfields are specified with a `.` character, such as foo.bar. + // see https://google.aip.dev/132#ordering for more details. + string order_by = 5; } // Response message for [MetadataService.ListExecutions][google.cloud.aiplatform.v1beta1.MetadataService.ListExecutions]. diff --git a/google/cloud/aiplatform/v1beta1/model_monitoring.proto b/google/cloud/aiplatform/v1beta1/model_monitoring.proto index 3dc687e5f1..955314ecd4 100644 --- a/google/cloud/aiplatform/v1beta1/model_monitoring.proto +++ b/google/cloud/aiplatform/v1beta1/model_monitoring.proto @@ -59,8 +59,8 @@ message ModelMonitoringObjectiveConfig { oneof data_source { // The resource name of the Dataset used to train this Model. string dataset = 3 [(google.api.resource_reference) = { - type: "aiplatform.googleapis.com/Dataset" - }]; + type: "aiplatform.googleapis.com/Dataset" + }]; // The Google Cloud Storage uri of the unmanaged Dataset used to train // this Model. @@ -135,10 +135,8 @@ message ModelMonitoringObjectiveConfig { // The config for integrating with Vertex Explainable AI. Only applicable if // the Model has explanation_spec populated. message ExplanationConfig { - // Output from - // [BatchPredictionJob][google.cloud.aiplatform.v1beta1.BatchPredictionJob] - // for Model Monitoring baseline dataset, which can be used to generate - // baseline attribution scores. + // Output from [BatchPredictionJob][google.cloud.aiplatform.v1beta1.BatchPredictionJob] for Model Monitoring baseline dataset, + // which can be used to generate baseline attribution scores. message ExplanationBaseline { // The storage format of the predictions generated BatchPrediction job. enum PredictionFormat { @@ -180,8 +178,7 @@ message ModelMonitoringObjectiveConfig { TrainingDataset training_dataset = 1; // The config for skew between training data and prediction data. - TrainingPredictionSkewDetectionConfig - training_prediction_skew_detection_config = 2; + TrainingPredictionSkewDetectionConfig training_prediction_skew_detection_config = 2; // The config for drift of prediction data. PredictionDriftDetectionConfig prediction_drift_detection_config = 3; diff --git a/google/cloud/aiplatform/v1beta1/pipeline_job.proto b/google/cloud/aiplatform/v1beta1/pipeline_job.proto index 9cb69a902e..cb450ee569 100644 --- a/google/cloud/aiplatform/v1beta1/pipeline_job.proto +++ b/google/cloud/aiplatform/v1beta1/pipeline_job.proto @@ -50,6 +50,18 @@ message PipelineJob { // The runtime config of a PipelineJob. message RuntimeConfig { + // The type of an input artifact. + message InputArtifact { + oneof kind { + // Artifact resource id from MLMD. Which is the last portion of an + // artifact resource + // name(projects/{project}/locations/{location}/metadataStores/default/artifacts/{artifact_id}). + // The artifact must stay within the same project, location and default + // metadatastore as the pipeline. + string artifact_id = 1; + } + } + // Deprecated. Use [RuntimeConfig.parameter_values][google.cloud.aiplatform.v1beta1.PipelineJob.RuntimeConfig.parameter_values] instead. The runtime // parameters of the PipelineJob. The parameters will be passed into // [PipelineJob.pipeline_spec][google.cloud.aiplatform.v1beta1.PipelineJob.pipeline_spec] to replace the placeholders at runtime. @@ -81,6 +93,10 @@ message PipelineJob { // will stop scheduling any new tasks when a task has failed. Any scheduled // tasks will continue to completion. PipelineFailurePolicy failure_policy = 4; + + // The runtime artifacts of the PipelineJob. The key will be the input + // artifact name and the value would be one of the InputArtifact. + map input_artifacts = 5; } // Output only. The resource name of the PipelineJob. diff --git a/google/cloud/aiplatform/v1beta1/pipeline_service.proto b/google/cloud/aiplatform/v1beta1/pipeline_service.proto index d66c78435c..8126d7b883 100644 --- a/google/cloud/aiplatform/v1beta1/pipeline_service.proto +++ b/google/cloud/aiplatform/v1beta1/pipeline_service.proto @@ -201,6 +201,9 @@ message ListTrainingPipelinesRequest { // * `training_task_definition` `=`, `!=` comparisons, and `:` wildcard. // * `create_time` supports `=`, `!=`,`<`, `<=`,`>`, `>=` comparisons. // `create_time` must be in RFC 3339 format. + // * `labels` supports general map functions that is: + // `labels.key=value` - key:value equality + // `labels.key:* - key existence // // Some examples of using the filter are: // @@ -366,6 +369,9 @@ message ListPipelineJobsRequest { // * `end_time` // * `start_time` string order_by = 6; + + // Mask specifying which fields to read. + google.protobuf.FieldMask read_mask = 7; } // Response message for [PipelineService.ListPipelineJobs][google.cloud.aiplatform.v1beta1.PipelineService.ListPipelineJobs] diff --git a/google/cloud/aiplatform/v1beta1/study.proto b/google/cloud/aiplatform/v1beta1/study.proto index b2595c684b..5ca0eb1a0e 100644 --- a/google/cloud/aiplatform/v1beta1/study.proto +++ b/google/cloud/aiplatform/v1beta1/study.proto @@ -485,6 +485,19 @@ message StudySpec { bool use_seconds = 5; } + // This contains flag for manually disabling transfer learning for a study. + // The names of prior studies being used for transfer learning (if any) + // are also listed here. + message TransferLearningConfig { + // Flag to to manually prevent vizier from using transfer learning on a + // new study. Otherwise, vizier will automatically determine whether or not + // to use transfer learning. + bool disable_transfer_learning = 1; + + // Output only. Names of previously completed studies + repeated string prior_study_names = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; + } + // The available search algorithms for the Study. enum Algorithm { // The default algorithm used by Vertex AI for [hyperparameter @@ -573,6 +586,10 @@ message StudySpec { // Describe which measurement selection type will be used MeasurementSelectionType measurement_selection_type = 7; + + // The configuration info/options for transfer learning. Currently supported + // for Vertex AI Vizier service, not HyperParameterTuningJob + TransferLearningConfig transfer_learning_config = 10; } // A message representing a Measurement of a Trial. A Measurement contains