googleapis/google/cloud/dataplex/v1/data_profile.proto

// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package google.cloud.dataplex.v1;

import "google/api/field_behavior.proto";
import "google/cloud/dataplex/v1/datascans_common.proto";
import "google/cloud/dataplex/v1/processing.proto";

option go_package = "cloud.google.com/go/dataplex/apiv1/dataplexpb;dataplexpb";
option java_multiple_files = true;
option java_outer_classname = "DataProfileProto";
option java_package = "com.google.cloud.dataplex.v1";

// DataProfileScan related setting.
message DataProfileSpec {
  // The configuration of post scan actions of DataProfileScan job.
  message PostScanActions {
    // The configuration of BigQuery export post scan action.
    message BigQueryExport {
      // Optional. The BigQuery table to export DataProfileScan results to.
      // Format:
      // //bigquery.googleapis.com/projects/PROJECT_ID/datasets/DATASET_ID/tables/TABLE_ID
      string results_table = 1 [(google.api.field_behavior) = OPTIONAL];
    }

    // Optional. If set, results will be exported to the provided BigQuery
    // table.
    BigQueryExport bigquery_export = 1 [(google.api.field_behavior) = OPTIONAL];
  }

  // The specification for fields to include or exclude in data profile scan.
  message SelectedFields {
    // Optional. Expected input is a list of fully qualified names of fields as
    // in the schema.
    //
    // Only top-level field names for nested fields are supported.
    // For instance, if 'x' is of nested field type, listing 'x' is supported
    // but 'x.y.z' is not supported. Here 'y' and 'y.z' are nested fields of
    // 'x'.
    repeated string field_names = 1 [(google.api.field_behavior) = OPTIONAL];
  }

  // Optional. The percentage of the records to be selected from the dataset for
  // DataScan.
  //
  // * Value can range between 0.0 and 100.0 with up to 3 significant decimal
  // digits.
  // * Sampling is not applied if `sampling_percent` is not specified, 0 or
  // 100.
  float sampling_percent = 2 [(google.api.field_behavior) = OPTIONAL];

  // Optional. A filter applied to all rows in a single DataScan job.
  // The filter needs to be a valid SQL expression for a WHERE clause in
  // BigQuery standard SQL syntax.
  // Example: col1 >= 0 AND col2 < 10
  string row_filter = 3 [(google.api.field_behavior) = OPTIONAL];

  // Optional. Actions to take upon job completion..
  PostScanActions post_scan_actions = 4
      [(google.api.field_behavior) = OPTIONAL];

  // Optional. The fields to include in data profile.
  //
  // If not specified, all fields at the time of profile scan job execution are
  // included, except for ones listed in `exclude_fields`.
  SelectedFields include_fields = 5 [(google.api.field_behavior) = OPTIONAL];

  // Optional. The fields to exclude from data profile.
  //
  // If specified, the fields will be excluded from data profile, regardless of
  // `include_fields` value.
  SelectedFields exclude_fields = 6 [(google.api.field_behavior) = OPTIONAL];

  // Optional. If set, the latest DataScan job result will be published as
  // Dataplex Universal Catalog metadata.
  bool catalog_publishing_enabled = 8 [(google.api.field_behavior) = OPTIONAL];
}

// DataProfileResult defines the output of DataProfileScan. Each field of the
// table will have field type specific profile result.
message DataProfileResult {
  // Contains name, type, mode and field type specific profile information.
  message Profile {
    // A field within a table.
    message Field {
      // The profile information for each field type.
      message ProfileInfo {
        // The profile information for a string type field.
        message StringFieldInfo {
          // Output only. Minimum length of non-null values in the scanned data.
          int64 min_length = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

          // Output only. Maximum length of non-null values in the scanned data.
          int64 max_length = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

          // Output only. Average length of non-null values in the scanned data.
          double average_length = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
        }

        // The profile information for an integer type field.
        message IntegerFieldInfo {
          // Output only. Average of non-null values in the scanned data. NaN,
          // if the field has a NaN.
          double average = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

          // Output only. Standard deviation of non-null values in the scanned
          // data. NaN, if the field has a NaN.
          double standard_deviation = 3
              [(google.api.field_behavior) = OUTPUT_ONLY];

          // Output only. Minimum of non-null values in the scanned data. NaN,
          // if the field has a NaN.
          int64 min = 4 [(google.api.field_behavior) = OUTPUT_ONLY];

          // Output only. A quartile divides the number of data points into four
          // parts, or quarters, of more-or-less equal size. Three main
          // quartiles used are: The first quartile (Q1) splits off the lowest
          // 25% of data from the highest 75%. It is also known as the lower or
          // 25th empirical quartile, as 25% of the data is below this point.
          // The second quartile (Q2) is the median of a data set. So, 50% of
          // the data lies below this point. The third quartile (Q3) splits off
          // the highest 25% of data from the lowest 75%. It is known as the
          // upper or 75th empirical quartile, as 75% of the data lies below
          // this point. Here, the quartiles is provided as an ordered list of
          // approximate quartile values for the scanned data, occurring in
          // order Q1, median, Q3.
          repeated int64 quartiles = 6
              [(google.api.field_behavior) = OUTPUT_ONLY];

          // Output only. Maximum of non-null values in the scanned data. NaN,
          // if the field has a NaN.
          int64 max = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
        }

        // The profile information for a double type field.
        message DoubleFieldInfo {
          // Output only. Average of non-null values in the scanned data. NaN,
          // if the field has a NaN.
          double average = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

          // Output only. Standard deviation of non-null values in the scanned
          // data. NaN, if the field has a NaN.
          double standard_deviation = 3
              [(google.api.field_behavior) = OUTPUT_ONLY];

          // Output only. Minimum of non-null values in the scanned data. NaN,
          // if the field has a NaN.
          double min = 4 [(google.api.field_behavior) = OUTPUT_ONLY];

          // Output only. A quartile divides the number of data points into four
          // parts, or quarters, of more-or-less equal size. Three main
          // quartiles used are: The first quartile (Q1) splits off the lowest
          // 25% of data from the highest 75%. It is also known as the lower or
          // 25th empirical quartile, as 25% of the data is below this point.
          // The second quartile (Q2) is the median of a data set. So, 50% of
          // the data lies below this point. The third quartile (Q3) splits off
          // the highest 25% of data from the lowest 75%. It is known as the
          // upper or 75th empirical quartile, as 75% of the data lies below
          // this point. Here, the quartiles is provided as an ordered list of
          // quartile values for the scanned data, occurring in order Q1,
          // median, Q3.
          repeated double quartiles = 6
              [(google.api.field_behavior) = OUTPUT_ONLY];

          // Output only. Maximum of non-null values in the scanned data. NaN,
          // if the field has a NaN.
          double max = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
        }

        // Top N non-null values in the scanned data.
        message TopNValue {
          // Output only. String value of a top N non-null value.
          string value = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

          // Output only. Count of the corresponding value in the scanned data.
          int64 count = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

          // Output only. Ratio of the corresponding value in the field against
          // the total number of rows in the scanned data.
          double ratio = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
        }

        // Output only. Ratio of rows with null value against total scanned
        // rows.
        double null_ratio = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

        // Output only. Ratio of rows with distinct values against total scanned
        // rows. Not available for complex non-groupable field type, including
        // RECORD, ARRAY, GEOGRAPHY, and JSON, as well as fields with REPEATABLE
        // mode.
        double distinct_ratio = 3 [(google.api.field_behavior) = OUTPUT_ONLY];

        // Output only. The list of top N non-null values, frequency and ratio
        // with which they occur in the scanned data. N is 10 or equal to the
        // number of distinct values in the field, whichever is smaller. Not
        // available for complex non-groupable field type, including RECORD,
        // ARRAY, GEOGRAPHY, and JSON, as well as fields with REPEATABLE mode.
        repeated TopNValue top_n_values = 4
            [(google.api.field_behavior) = OUTPUT_ONLY];

        // Structural and profile information for specific field type. Not
        // available, if mode is REPEATABLE.
        oneof field_info {
          // String type field information.
          StringFieldInfo string_profile = 101;

          // Integer type field information.
          IntegerFieldInfo integer_profile = 102;

          // Double type field information.
          DoubleFieldInfo double_profile = 103;
        }
      }

      // Output only. The name of the field.
      string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

      // Output only. The data type retrieved from the schema of the data
      // source. For instance, for a BigQuery native table, it is the [BigQuery
      // Table
      // Schema](https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#tablefieldschema).
      // For a Dataplex Universal Catalog Entity, it is the [Entity
      // Schema](https://cloud.google.com/dataplex/docs/reference/rpc/google.cloud.dataplex.v1#type_3).
      string type = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

      // Output only. The mode of the field. Possible values include:
      //
      // * REQUIRED, if it is a required field.
      // * NULLABLE, if it is an optional field.
      // * REPEATED, if it is a repeated field.
      string mode = 3 [(google.api.field_behavior) = OUTPUT_ONLY];

      // Output only. Profile information for the corresponding field.
      ProfileInfo profile = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
    }

    // Output only. List of fields with structural and profile information for
    // each field.
    repeated Field fields = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // The result of post scan actions of DataProfileScan job.
  message PostScanActionsResult {
    // The result of BigQuery export post scan action.
    message BigQueryExportResult {
      // Execution state for the exporting.
      enum State {
        // The exporting state is unspecified.
        STATE_UNSPECIFIED = 0;

        // The exporting completed successfully.
        SUCCEEDED = 1;

        // The exporting is no longer running due to an error.
        FAILED = 2;

        // The exporting is skipped due to no valid scan result to export
        // (usually caused by scan failed).
        SKIPPED = 3;
      }

      // Output only. Execution state for the BigQuery exporting.
      State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

      // Output only. Additional information about the BigQuery exporting.
      string message = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
    }

    // Output only. The result of BigQuery export post scan action.
    BigQueryExportResult bigquery_export_result = 1
        [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // Output only. The count of rows scanned.
  int64 row_count = 3 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The profile information per field.
  Profile profile = 4 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The data scanned for this result.
  ScannedData scanned_data = 5 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The result of post scan actions.
  PostScanActionsResult post_scan_actions_result = 6
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The status of publishing the data scan as Dataplex Universal
  // Catalog metadata.
  DataScanCatalogPublishingStatus catalog_publishing_status = 7
      [(google.api.field_behavior) = OUTPUT_ONLY];
}