dataplex/v1/logs.proto

// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package google.cloud.dataplex.v1;

import "google/api/field_behavior.proto";
import "google/api/resource.proto";
import "google/protobuf/duration.proto";
import "google/protobuf/timestamp.proto";

option go_package = "cloud.google.com/go/dataplex/apiv1/dataplexpb;dataplexpb";
option java_multiple_files = true;
option java_outer_classname = "LogsProto";
option java_package = "com.google.cloud.dataplex.v1";

// The payload associated with Discovery data processing.
message DiscoveryEvent {
  // The type of the event.
  enum EventType {
    // An unspecified event type.
    EVENT_TYPE_UNSPECIFIED = 0;

    // An event representing discovery configuration in effect.
    CONFIG = 1;

    // An event representing a metadata entity being created.
    ENTITY_CREATED = 2;

    // An event representing a metadata entity being updated.
    ENTITY_UPDATED = 3;

    // An event representing a metadata entity being deleted.
    ENTITY_DELETED = 4;

    // An event representing a partition being created.
    PARTITION_CREATED = 5;

    // An event representing a partition being updated.
    PARTITION_UPDATED = 6;

    // An event representing a partition being deleted.
    PARTITION_DELETED = 7;
  }

  // The type of the entity.
  enum EntityType {
    // An unspecified event type.
    ENTITY_TYPE_UNSPECIFIED = 0;

    // Entities representing structured data.
    TABLE = 1;

    // Entities representing unstructured data.
    FILESET = 2;
  }

  // Details about configuration events.
  message ConfigDetails {
    // A list of discovery configuration parameters in effect.
    // The keys are the field paths within DiscoverySpec.
    // Eg. includePatterns, excludePatterns, csvOptions.disableTypeInference,
    // etc.
    map<string, string> parameters = 1;
  }

  // Details about the entity.
  message EntityDetails {
    // The name of the entity resource.
    // The name is the fully-qualified resource name.
    string entity = 1;

    // The type of the entity resource.
    EntityType type = 2;
  }

  // Details about the partition.
  message PartitionDetails {
    // The name to the partition resource.
    // The name is the fully-qualified resource name.
    string partition = 1;

    // The name to the containing entity resource.
    // The name is the fully-qualified resource name.
    string entity = 2;

    // The type of the containing entity resource.
    EntityType type = 3;

    // The locations of the data items (e.g., a Cloud Storage objects) sampled
    // for metadata inference.
    repeated string sampled_data_locations = 4;
  }

  // Details about the action.
  message ActionDetails {
    // The type of action.
    // Eg. IncompatibleDataSchema, InvalidDataFormat
    string type = 1;
  }

  // The log message.
  string message = 1;

  // The id of the associated lake.
  string lake_id = 2;

  // The id of the associated zone.
  string zone_id = 3;

  // The id of the associated asset.
  string asset_id = 4;

  // The data location associated with the event.
  string data_location = 5;

  // The type of the event being logged.
  EventType type = 10;

  // Additional details about the event.
  oneof details {
    // Details about discovery configuration in effect.
    ConfigDetails config = 20;

    // Details about the entity associated with the event.
    EntityDetails entity = 21;

    // Details about the partition associated with the event.
    PartitionDetails partition = 22;

    // Details about the action associated with the event.
    ActionDetails action = 23;
  }
}

// The payload associated with Job logs that contains events describing jobs
// that have run within a Lake.
message JobEvent {
  // The type of the job.
  enum Type {
    // Unspecified job type.
    TYPE_UNSPECIFIED = 0;

    // Spark jobs.
    SPARK = 1;

    // Notebook jobs.
    NOTEBOOK = 2;
  }

  // The completion status of the job.
  enum State {
    // Unspecified job state.
    STATE_UNSPECIFIED = 0;

    // Job successfully completed.
    SUCCEEDED = 1;

    // Job was unsuccessful.
    FAILED = 2;

    // Job was cancelled by the user.
    CANCELLED = 3;

    // Job was cancelled or aborted via the service executing the job.
    ABORTED = 4;
  }

  // The service used to execute the job.
  enum Service {
    // Unspecified service.
    SERVICE_UNSPECIFIED = 0;

    // Cloud Dataproc.
    DATAPROC = 1;
  }

  // Job Execution trigger.
  enum ExecutionTrigger {
    // The job execution trigger is unspecified.
    EXECUTION_TRIGGER_UNSPECIFIED = 0;

    // The job was triggered by Dataplex based on trigger spec from task
    // definition.
    TASK_CONFIG = 1;

    // The job was triggered by the explicit call of Task API.
    RUN_REQUEST = 2;
  }

  // The log message.
  string message = 1;

  // The unique id identifying the job.
  string job_id = 2;

  // The time when the job started running.
  google.protobuf.Timestamp start_time = 3;

  // The time when the job ended running.
  google.protobuf.Timestamp end_time = 4;

  // The job state on completion.
  State state = 5;

  // The number of retries.
  int32 retries = 6;

  // The type of the job.
  Type type = 7;

  // The service used to execute the job.
  Service service = 8;

  // The reference to the job within the service.
  string service_job = 9;

  // Job execution trigger.
  ExecutionTrigger execution_trigger = 11;
}

// These messages contain information about sessions within an environment.
// The monitored resource is 'Environment'.
message SessionEvent {
  // The type of the event.
  enum EventType {
    // An unspecified event type.
    EVENT_TYPE_UNSPECIFIED = 0;

    // Event when the session is assigned to a user.
    START = 1;

    // Event for stop of a session.
    STOP = 2;

    // Query events in the session.
    QUERY = 3;

    // Event for creation of a cluster. It is not yet assigned to a user.
    // This comes before START in the sequence
    CREATE = 4;
  }

  // Execution details of the query.
  message QueryDetail {
    // Query Execution engine.
    enum Engine {
      // An unspecified Engine type.
      ENGINE_UNSPECIFIED = 0;

      // Spark-sql engine is specified in Query.
      SPARK_SQL = 1;

      // BigQuery engine is specified in Query.
      BIGQUERY = 2;
    }

    // The unique Query id identifying the query.
    string query_id = 1;

    // The query text executed.
    string query_text = 2;

    // Query Execution engine.
    Engine engine = 3;

    // Time taken for execution of the query.
    google.protobuf.Duration duration = 4;

    // The size of results the query produced.
    int64 result_size_bytes = 5;

    // The data processed by the query.
    int64 data_processed_bytes = 6;
  }

  // The log message.
  string message = 1;

  // The information about the user that created the session. It will be the
  // email address of the user.
  string user_id = 2;

  // Unique identifier for the session.
  string session_id = 3;

  // The type of the event.
  EventType type = 4;

  // Additional information about the Query metadata.
  oneof detail {
    // The execution details of the query.
    QueryDetail query = 5;
  }

  // The status of the event.
  bool event_succeeded = 6;

  // If the session is associated with an environment with fast startup enabled,
  // and was created before being assigned to a user.
  bool fast_startup_enabled = 7;

  // The idle duration of a warm pooled session before it is assigned to user.
  google.protobuf.Duration unassigned_duration = 8;
}

// Payload associated with Governance related log events.
message GovernanceEvent {
  // Information about Entity resource that the log event is associated with.
  message Entity {
    // Type of entity.
    enum EntityType {
      // An unspecified Entity type.
      ENTITY_TYPE_UNSPECIFIED = 0;

      // Table entity type.
      TABLE = 1;

      // Fileset entity type.
      FILESET = 2;
    }

    // The Entity resource the log event is associated with.
    // Format:
    // `projects/{project_number}/locations/{location_id}/lakes/{lake_id}/zones/{zone_id}/entities/{entity_id}`
    string entity = 1 [(google.api.resource_reference) = {
      type: "dataplex.googleapis.com/Entity"
    }];

    // Type of entity.
    EntityType entity_type = 2;
  }

  // Type of governance log event.
  enum EventType {
    // An unspecified event type.
    EVENT_TYPE_UNSPECIFIED = 0;

    // Resource IAM policy update event.
    RESOURCE_IAM_POLICY_UPDATE = 1;

    // BigQuery table create event.
    BIGQUERY_TABLE_CREATE = 2;

    // BigQuery table update event.
    BIGQUERY_TABLE_UPDATE = 3;

    // BigQuery table delete event.
    BIGQUERY_TABLE_DELETE = 4;

    // BigQuery connection create event.
    BIGQUERY_CONNECTION_CREATE = 5;

    // BigQuery connection update event.
    BIGQUERY_CONNECTION_UPDATE = 6;

    // BigQuery connection delete event.
    BIGQUERY_CONNECTION_DELETE = 7;

    // BigQuery taxonomy created.
    BIGQUERY_TAXONOMY_CREATE = 10;

    // BigQuery policy tag created.
    BIGQUERY_POLICY_TAG_CREATE = 11;

    // BigQuery policy tag deleted.
    BIGQUERY_POLICY_TAG_DELETE = 12;

    // BigQuery set iam policy for policy tag.
    BIGQUERY_POLICY_TAG_SET_IAM_POLICY = 13;

    // Access policy update event.
    ACCESS_POLICY_UPDATE = 14;

    // Number of resources matched with particular Query.
    GOVERNANCE_RULE_MATCHED_RESOURCES = 15;

    // Rule processing exceeds the allowed limit.
    GOVERNANCE_RULE_SEARCH_LIMIT_EXCEEDS = 16;

    // Rule processing errors.
    GOVERNANCE_RULE_ERRORS = 17;

    // Governance rule processing Event.
    GOVERNANCE_RULE_PROCESSING = 18;
  }

  // The log message.
  string message = 1;

  // The type of the event.
  EventType event_type = 2;

  // Entity resource information if the log event is associated with a
  // specific entity.
  optional Entity entity = 3;
}

// These messages contain information about the execution of a datascan.
// The monitored resource is 'DataScan'
// Next ID: 13
message DataScanEvent {
  // The type of the data scan.
  enum ScanType {
    // An unspecified data scan type.
    SCAN_TYPE_UNSPECIFIED = 0;

    // Data scan for data profile.
    DATA_PROFILE = 1;

    // Data scan for data quality.
    DATA_QUALITY = 2;
  }

  // The job state of the data scan.
  enum State {
    // Unspecified job state.
    STATE_UNSPECIFIED = 0;

    // Data scan job started.
    STARTED = 1;

    // Data scan job successfully completed.
    SUCCEEDED = 2;

    // Data scan job was unsuccessful.
    FAILED = 3;

    // Data scan job was cancelled.
    CANCELLED = 4;

    // Data scan job was createed.
    CREATED = 5;
  }

  // The trigger type for the data scan.
  enum Trigger {
    // An unspecified trigger type.
    TRIGGER_UNSPECIFIED = 0;

    // Data scan triggers on demand.
    ON_DEMAND = 1;

    // Data scan triggers as per schedule.
    SCHEDULE = 2;
  }

  // The scope of job for the data scan.
  enum Scope {
    // An unspecified scope type.
    SCOPE_UNSPECIFIED = 0;

    // Data scan runs on all of the data.
    FULL = 1;

    // Data scan runs on incremental data.
    INCREMENTAL = 2;
  }

  // Data profile result for data scan job.
  message DataProfileResult {
    // The count of rows processed in the data scan job.
    int64 row_count = 1;
  }

  // Data quality result for data scan job.
  message DataQualityResult {
    // The count of rows processed in the data scan job.
    int64 row_count = 1;

    // Whether the data quality result was `pass` or not.
    bool passed = 2;

    // The result of each dimension for data quality result.
    // The key of the map is the name of the dimension.
    // The value is the bool value depicting whether the dimension result was
    // `pass` or not.
    map<string, bool> dimension_passed = 3;

    // The table-level data quality score for the data scan job.
    //
    // The data quality score ranges between [0, 100] (up to two decimal
    // points).
    float score = 4;

    // The score of each dimension for data quality result.
    // The key of the map is the name of the dimension.
    // The value is the data quality score for the dimension.
    //
    // The score ranges between [0, 100] (up to two decimal
    // points).
    map<string, float> dimension_score = 5;

    // The score of each column scanned in the data scan job.
    // The key of the map is the name of the column.
    // The value is the data quality score for the column.
    //
    // The score ranges between [0, 100] (up to two decimal
    // points).
    map<string, float> column_score = 6;
  }

  // Applied configs for data profile type data scan job.
  message DataProfileAppliedConfigs {
    // The percentage of the records selected from the dataset for DataScan.
    //
    // * Value ranges between 0.0 and 100.0.
    // * Value 0.0 or 100.0 imply that sampling was not applied.
    float sampling_percent = 1;

    // Boolean indicating whether a row filter was applied in the DataScan job.
    bool row_filter_applied = 2;

    // Boolean indicating whether a column filter was applied in the DataScan
    // job.
    bool column_filter_applied = 3;
  }

  // Applied configs for data quality type data scan job.
  message DataQualityAppliedConfigs {
    // The percentage of the records selected from the dataset for DataScan.
    //
    // * Value ranges between 0.0 and 100.0.
    // * Value 0.0 or 100.0 imply that sampling was not applied.
    float sampling_percent = 1;

    // Boolean indicating whether a row filter was applied in the DataScan job.
    bool row_filter_applied = 2;
  }

  // Post scan actions result for data scan job.
  message PostScanActionsResult {
    // The result of BigQuery export post scan action.
    message BigQueryExportResult {
      // Execution state for the exporting.
      enum State {
        // The exporting state is unspecified.
        STATE_UNSPECIFIED = 0;

        // The exporting completed successfully.
        SUCCEEDED = 1;

        // The exporting is no longer running due to an error.
        FAILED = 2;

        // The exporting is skipped due to no valid scan result to export
        // (usually caused by scan failed).
        SKIPPED = 3;
      }

      // Execution state for the BigQuery exporting.
      State state = 1;

      // Additional information about the BigQuery exporting.
      string message = 2;
    }

    // The result of BigQuery export post scan action.
    BigQueryExportResult bigquery_export_result = 1;
  }

  // The data source of the data scan
  string data_source = 1;

  // The identifier of the specific data scan job this log entry is for.
  string job_id = 2;

  // The time when the data scan job was created.
  google.protobuf.Timestamp create_time = 12;

  // The time when the data scan job started to run.
  google.protobuf.Timestamp start_time = 3;

  // The time when the data scan job finished.
  google.protobuf.Timestamp end_time = 4;

  // The type of the data scan.
  ScanType type = 5;

  // The status of the data scan job.
  State state = 6;

  // The message describing the data scan job event.
  string message = 7;

  // A version identifier of the spec which was used to execute this job.
  string spec_version = 8;

  // The trigger type of the data scan job.
  Trigger trigger = 9;

  // The scope of the data scan (e.g. full, incremental).
  Scope scope = 10;

  // The result of the data scan job.
  oneof result {
    // Data profile result for data profile type data scan.
    DataProfileResult data_profile = 101;

    // Data quality result for data quality type data scan.
    DataQualityResult data_quality = 102;
  }

  // The applied configs in the data scan job.
  oneof appliedConfigs {
    // Applied configs for data profile type data scan.
    DataProfileAppliedConfigs data_profile_configs = 201;

    // Applied configs for data quality type data scan.
    DataQualityAppliedConfigs data_quality_configs = 202;
  }

  // The result of post scan actions.
  PostScanActionsResult post_scan_actions_result = 11;
}

// Information about the result of a data quality rule for data quality scan.
// The monitored resource is 'DataScan'.
message DataQualityScanRuleResult {
  // The type of the data quality rule.
  enum RuleType {
    // An unspecified rule type.
    RULE_TYPE_UNSPECIFIED = 0;

    // Please see
    // https://cloud.google.com/dataplex/docs/reference/rest/v1/DataQualityRule#nonnullexpectation.
    NON_NULL_EXPECTATION = 1;

    // Please see
    // https://cloud.google.com/dataplex/docs/reference/rest/v1/DataQualityRule#rangeexpectation.
    RANGE_EXPECTATION = 2;

    // Please see
    // https://cloud.google.com/dataplex/docs/reference/rest/v1/DataQualityRule#regexexpectation.
    REGEX_EXPECTATION = 3;

    // Please see
    // https://cloud.google.com/dataplex/docs/reference/rest/v1/DataQualityRule#rowconditionexpectation.
    ROW_CONDITION_EXPECTATION = 4;

    // Please see
    // https://cloud.google.com/dataplex/docs/reference/rest/v1/DataQualityRule#setexpectation.
    SET_EXPECTATION = 5;

    // Please see
    // https://cloud.google.com/dataplex/docs/reference/rest/v1/DataQualityRule#statisticrangeexpectation.
    STATISTIC_RANGE_EXPECTATION = 6;

    // Please see
    // https://cloud.google.com/dataplex/docs/reference/rest/v1/DataQualityRule#tableconditionexpectation.
    TABLE_CONDITION_EXPECTATION = 7;

    // Please see
    // https://cloud.google.com/dataplex/docs/reference/rest/v1/DataQualityRule#uniquenessexpectation.
    UNIQUENESS_EXPECTATION = 8;
  }

  // The evaluation type of the data quality rule.
  enum EvaluationType {
    // An unspecified evaluation type.
    EVALUATION_TYPE_UNSPECIFIED = 0;

    // The rule evaluation is done at per row level.
    PER_ROW = 1;

    // The rule evaluation is done for an aggregate of rows.
    AGGREGATE = 2;
  }

  // Whether the data quality rule passed or failed.
  enum Result {
    // An unspecified result.
    RESULT_UNSPECIFIED = 0;

    // The data quality rule passed.
    PASSED = 1;

    // The data quality rule failed.
    FAILED = 2;
  }

  // Identifier of the specific data scan job this log entry is for.
  string job_id = 1;

  // The data source of the data scan (e.g. BigQuery table name).
  string data_source = 2;

  // The column which this rule is evaluated against.
  string column = 3;

  // The name of the data quality rule.
  string rule_name = 4;

  // The type of the data quality rule.
  RuleType rule_type = 5;

  // The evaluation type of the data quality rule.
  EvaluationType evalution_type = 6;

  // The dimension of the data quality rule.
  string rule_dimension = 7;

  // The passing threshold ([0.0, 100.0]) of the data quality rule.
  double threshold_percent = 8;

  // The result of the data quality rule.
  Result result = 9;

  // The number of rows evaluated against the data quality rule.
  // This field is only valid for rules of PER_ROW evaluation type.
  int64 evaluated_row_count = 10;

  // The number of rows which passed a rule evaluation.
  // This field is only valid for rules of PER_ROW evaluation type.
  int64 passed_row_count = 11;

  // The number of rows with null values in the specified column.
  int64 null_row_count = 12;
}