datastream/v1alpha1/datastream_resources.proto

// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package google.cloud.datastream.v1alpha1;

import "google/api/field_behavior.proto";
import "google/api/resource.proto";
import "google/protobuf/duration.proto";
import "google/protobuf/timestamp.proto";

option go_package = "cloud.google.com/go/datastream/apiv1alpha1/datastreampb;datastreampb";
option java_multiple_files = true;
option java_outer_classname = "CloudDatastreamResourcesProto";
option java_package = "com.google.cloud.datastream.v1alpha1";
option csharp_namespace = "Google.Cloud.Datastream.V1Alpha1";
option php_namespace = "Google\\Cloud\\Datastream\\V1alpha1";
option ruby_package = "Google::Cloud::Datastream::V1alpha1";

// Oracle database profile.
message OracleProfile {
  // Required. Hostname for the Oracle connection.
  string hostname = 1 [(google.api.field_behavior) = REQUIRED];

  // Port for the Oracle connection, default value is 1521.
  int32 port = 2;

  // Required. Username for the Oracle connection.
  string username = 3 [(google.api.field_behavior) = REQUIRED];

  // Required. Password for the Oracle connection.
  string password = 4 [(google.api.field_behavior) = REQUIRED];

  // Required. Database for the Oracle connection.
  string database_service = 5 [(google.api.field_behavior) = REQUIRED];

  // Connection string attributes
  map<string, string> connection_attributes = 6;
}

// MySQL database profile.
message MysqlProfile {
  // Required. Hostname for the MySQL connection.
  string hostname = 1 [(google.api.field_behavior) = REQUIRED];

  // Port for the MySQL connection, default value is 3306.
  int32 port = 2;

  // Required. Username for the MySQL connection.
  string username = 3 [(google.api.field_behavior) = REQUIRED];

  // Required. Input only. Password for the MySQL connection.
  string password = 4 [
    (google.api.field_behavior) = REQUIRED,
    (google.api.field_behavior) = INPUT_ONLY
  ];

  // SSL configuration for the MySQL connection.
  MysqlSslConfig ssl_config = 5;
}

// Cloud Storage bucket profile.
message GcsProfile {
  // Required. The full project and resource path for Cloud Storage bucket including the
  // name.
  string bucket_name = 1 [(google.api.field_behavior) = REQUIRED];

  // The root path inside the Cloud Storage bucket.
  string root_path = 2;
}

// No connectivity settings.
message NoConnectivitySettings {

}

// Static IP address connectivity.
message StaticServiceIpConnectivity {

}

// Forward SSH Tunnel connectivity.
message ForwardSshTunnelConnectivity {
  // Required. Hostname for the SSH tunnel.
  string hostname = 1 [(google.api.field_behavior) = REQUIRED];

  // Required. Username for the SSH tunnel.
  string username = 2 [(google.api.field_behavior) = REQUIRED];

  // Port for the SSH tunnel, default value is 22.
  int32 port = 3;

  oneof authentication_method {
    // Input only. SSH password.
    string password = 100 [(google.api.field_behavior) = INPUT_ONLY];

    // Input only. SSH private key.
    string private_key = 101 [(google.api.field_behavior) = INPUT_ONLY];
  }
}

// The VPC Peering configuration is used to create VPC peering between
// Datastream and the consumer's VPC.
message VpcPeeringConfig {
  // Required. fully qualified name of the VPC Datastream will peer to.
  string vpc_name = 1 [(google.api.field_behavior) = REQUIRED];

  // Required. A free subnet for peering. (CIDR of /29)
  string subnet = 2 [(google.api.field_behavior) = REQUIRED];
}

// The PrivateConnection resource is used to establish private connectivity
// between Datastream and a customer's network.
message PrivateConnection {
  option (google.api.resource) = {
    type: "datastream.googleapis.com/PrivateConnection"
    pattern: "projects/{project}/locations/{location}/privateConnections/{private_connection}"
  };

  // Private Connection state.
  enum State {
    STATE_UNSPECIFIED = 0;

    // The private connection is in creation state - creating resources.
    CREATING = 1;

    // The private connection has been created with all of it's resources.
    CREATED = 2;

    // The private connection creation has failed.
    FAILED = 3;
  }

  // Output only. The resource's name.
  string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The create time of the resource.
  google.protobuf.Timestamp create_time = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The update time of the resource.
  google.protobuf.Timestamp update_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Labels.
  map<string, string> labels = 4;

  // Required. Display name.
  string display_name = 5 [(google.api.field_behavior) = REQUIRED];

  // Output only. The state of the Private Connection.
  State state = 6 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. In case of error, the details of the error in a user-friendly format.
  Error error = 7 [(google.api.field_behavior) = OUTPUT_ONLY];

  // VPC Peering Config
  VpcPeeringConfig vpc_peering_config = 100;
}

// Private Connectivity
message PrivateConnectivity {
  string private_connection_name = 1 [(google.api.field_behavior) = REQUIRED];
}

// The Route resource is the child of the PrivateConnection resource.
// It used to define a route for a PrivateConnection setup.
message Route {
  option (google.api.resource) = {
    type: "datastream.googleapis.com/Route"
    pattern: "projects/{project}/locations/{location}/privateConnections/{private_connection}/routes/{route}"
  };

  // Output only. The resource's name.
  string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The create time of the resource.
  google.protobuf.Timestamp create_time = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The update time of the resource.
  google.protobuf.Timestamp update_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Labels.
  map<string, string> labels = 4;

  // Required. Display name.
  string display_name = 5 [(google.api.field_behavior) = REQUIRED];

  // Required. Destination address for connection
  string destination_address = 6 [(google.api.field_behavior) = REQUIRED];

  // Destination port for connection
  int32 destination_port = 7;
}

// MySQL SSL configuration information.
message MysqlSslConfig {
  // Input only. PEM-encoded private key associated with the Client Certificate.
  // If this field is used then the 'client_certificate' and the
  // 'ca_certificate' fields are mandatory.
  string client_key = 11 [(google.api.field_behavior) = INPUT_ONLY];

  // Output only. Indicates whether the client_key field is set.
  bool client_key_set = 12 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Input only. PEM-encoded certificate that will be used by the replica to
  // authenticate against the source database server. If this field is used
  // then the 'client_key' and the 'ca_certificate' fields are mandatory.
  string client_certificate = 13 [(google.api.field_behavior) = INPUT_ONLY];

  // Output only. Indicates whether the client_certificate field is set.
  bool client_certificate_set = 14 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Input only. PEM-encoded certificate of the CA that signed the source database
  // server's certificate.
  string ca_certificate = 15 [(google.api.field_behavior) = INPUT_ONLY];

  // Output only. Indicates whether the ca_certificate field is set.
  bool ca_certificate_set = 16 [(google.api.field_behavior) = OUTPUT_ONLY];
}

message ConnectionProfile {
  option (google.api.resource) = {
    type: "datastream.googleapis.com/ConnectionProfile"
    pattern: "projects/{project}/locations/{location}/connectionProfiles/{connection_profile}"
  };

  // Output only. The resource's name.
  string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The create time of the resource.
  google.protobuf.Timestamp create_time = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The update time of the resource.
  google.protobuf.Timestamp update_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Labels.
  map<string, string> labels = 4;

  // Required. Display name.
  string display_name = 5 [(google.api.field_behavior) = REQUIRED];

  // Connection configuration for the ConnectionProfile.
  oneof profile {
    // Oracle ConnectionProfile configuration.
    OracleProfile oracle_profile = 100;

    // Cloud Storage ConnectionProfile configuration.
    GcsProfile gcs_profile = 101;

    // MySQL ConnectionProfile configuration.
    MysqlProfile mysql_profile = 102;
  }

  // Connectivity options used to establish a connection to the profile.
  oneof connectivity {
    // No connectivity option chosen.
    NoConnectivitySettings no_connectivity = 200;

    // Static Service IP connectivity.
    StaticServiceIpConnectivity static_service_ip_connectivity = 201;

    // Forward SSH tunnel connectivity.
    ForwardSshTunnelConnectivity forward_ssh_connectivity = 202;

    // Private connectivity.
    PrivateConnectivity private_connectivity = 203;
  }
}

// Oracle Column.
message OracleColumn {
  // Column name.
  string column_name = 1;

  // The Oracle data type.
  string data_type = 2;

  // Column length.
  int32 length = 3;

  // Column precision.
  int32 precision = 4;

  // Column scale.
  int32 scale = 5;

  // Column encoding.
  string encoding = 6;

  // Whether or not the column represents a primary key.
  bool primary_key = 7;

  // Whether or not the column can accept a null value.
  bool nullable = 8;

  // The ordinal position of the column in the table.
  int32 ordinal_position = 9;
}

// Oracle table.
message OracleTable {
  // Table name.
  string table_name = 1;

  // Oracle columns in the schema.
  // When unspecified as part of inclue/exclude lists, includes/excludes
  // everything.
  repeated OracleColumn oracle_columns = 2;
}

// Oracle schema.
message OracleSchema {
  // Schema name.
  string schema_name = 1;

  // Tables in the schema.
  repeated OracleTable oracle_tables = 2;
}

// Oracle database structure.
message OracleRdbms {
  // Oracle schemas/databases in the database server.
  repeated OracleSchema oracle_schemas = 1;
}

// Oracle data source configuration
message OracleSourceConfig {
  // Oracle objects to include in the stream.
  OracleRdbms allowlist = 1;

  // Oracle objects to exclude from the stream.
  OracleRdbms rejectlist = 2;
}

// MySQL Column.
message MysqlColumn {
  // Column name.
  string column_name = 1;

  // The MySQL data type. Full data types list can be found here:
  // https://dev.mysql.com/doc/refman/8.0/en/data-types.html
  string data_type = 2;

  // Column length.
  int32 length = 3;

  // Column collation.
  string collation = 4;

  // Whether or not the column represents a primary key.
  bool primary_key = 5;

  // Whether or not the column can accept a null value.
  bool nullable = 6;

  // The ordinal position of the column in the table.
  int32 ordinal_position = 7;
}

// MySQL table.
message MysqlTable {
  // Table name.
  string table_name = 1;

  // MySQL columns in the database.
  // When unspecified as part of include/exclude lists, includes/excludes
  // everything.
  repeated MysqlColumn mysql_columns = 2;
}

// MySQL database.
message MysqlDatabase {
  // Database name.
  string database_name = 1;

  // Tables in the database.
  repeated MysqlTable mysql_tables = 2;
}

// MySQL database structure
message MysqlRdbms {
  // Mysql databases on the server
  repeated MysqlDatabase mysql_databases = 1;
}

// MySQL source configuration
message MysqlSourceConfig {
  // MySQL objects to retrieve from the source.
  MysqlRdbms allowlist = 1;

  // MySQL objects to exclude from the stream.
  MysqlRdbms rejectlist = 2;
}

// The configuration of the stream source.
message SourceConfig {
  // Required. Source connection profile identifier.
  string source_connection_profile_name = 1 [(google.api.field_behavior) = REQUIRED];

  // Stream configuration that is specific to the data source type.
  oneof source_stream_config {
    // Oracle data source configuration
    OracleSourceConfig oracle_source_config = 100;

    // MySQL data source configuration
    MysqlSourceConfig mysql_source_config = 101;
  }
}

// AVRO file format configuration.
message AvroFileFormat {

}

// File format in Cloud Storage.
enum GcsFileFormat {
  option deprecated = true;

  // Unspecified Cloud Storage file format.
  GCS_FILE_FORMAT_UNSPECIFIED = 0;

  // Avro file format
  AVRO = 1;
}

// Schema file format.
enum SchemaFileFormat {
  // Unspecified schema file format.
  SCHEMA_FILE_FORMAT_UNSPECIFIED = 0;

  // Do not attach schema file.
  NO_SCHEMA_FILE = 1;

  // Avro schema format.
  AVRO_SCHEMA_FILE = 2;
}

// JSON file format configuration.
message JsonFileFormat {
  // Json file compression.
  enum JsonCompression {
    // Unspecified json file compression.
    JSON_COMPRESSION_UNSPECIFIED = 0;

    // Do not compress JSON file.
    NO_COMPRESSION = 1;

    // Gzip compression.
    GZIP = 2;
  }

  // The schema file format along JSON data files.
  SchemaFileFormat schema_file_format = 1;

  // Compression of the loaded JSON file.
  JsonCompression compression = 2;
}

// Google Cloud Storage destination configuration
message GcsDestinationConfig {
  // Path inside the Cloud Storage bucket to write data to.
  string path = 1;

  // File format that data should be written in.
  // Deprecated field - use file_format instead.
  GcsFileFormat gcs_file_format = 2 [deprecated = true];

  // The maximum file size to be saved in the bucket.
  int32 file_rotation_mb = 3;

  // The maximum duration for which new events are added before a file is
  // closed and a new file is created.
  google.protobuf.Duration file_rotation_interval = 4;

  // File Format that the data should be written in.
  oneof file_format {
    // AVRO file format configuration.
    AvroFileFormat avro_file_format = 100;

    // JSON file format configuration.
    JsonFileFormat json_file_format = 101;
  }
}

// The configuration of the stream destination.
message DestinationConfig {
  // Required. Destination connection profile identifier.
  string destination_connection_profile_name = 1 [(google.api.field_behavior) = REQUIRED];

  // Stream configuration that is specific to the data destination type.
  oneof destination_stream_config {
    GcsDestinationConfig gcs_destination_config = 100;
  }
}

message Stream {
  option (google.api.resource) = {
    type: "datastream.googleapis.com/Stream"
    pattern: "projects/{project}/locations/{location}/streams/{stream}"
  };

  // Backfill strategy to automatically backfill the Stream's objects.
  // Specific objects can be excluded.
  message BackfillAllStrategy {
    // List of objects to exclude.
    oneof excluded_objects {
      // Oracle data source objects to avoid backfilling.
      OracleRdbms oracle_excluded_objects = 1;

      // MySQL data source objects to avoid backfilling.
      MysqlRdbms mysql_excluded_objects = 2;
    }
  }

  // Backfill strategy to disable automatic backfill for the Stream's objects.
  message BackfillNoneStrategy {

  }

  // Stream state.
  enum State {
    // Unspecified stream state.
    STATE_UNSPECIFIED = 0;

    // The stream has been created.
    CREATED = 1;

    // The stream is running.
    RUNNING = 2;

    // The stream is paused.
    PAUSED = 3;

    // The stream is in maintenance mode.
    //
    // Updates are rejected on the resource in this state.
    MAINTENANCE = 4;

    // The stream is experiencing an error that is preventing data from being
    // streamed.
    FAILED = 5;

    // The stream has experienced a terminal failure.
    FAILED_PERMANENTLY = 6;

    // The stream is starting, but not yet running.
    STARTING = 7;

    // The Stream is no longer reading new events, but still writing events in
    // the buffer.
    DRAINING = 8;
  }

  // Output only. The stream's name.
  string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The creation time of the stream.
  google.protobuf.Timestamp create_time = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The last update time of the stream.
  google.protobuf.Timestamp update_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Labels.
  map<string, string> labels = 4;

  // Required. Display name.
  string display_name = 5 [(google.api.field_behavior) = REQUIRED];

  // Required. Source connection profile configuration.
  SourceConfig source_config = 6 [(google.api.field_behavior) = REQUIRED];

  // Required. Destination connection profile configuration.
  DestinationConfig destination_config = 7 [(google.api.field_behavior) = REQUIRED];

  // The state of the stream.
  State state = 8;

  // Stream backfill strategy.
  oneof backfill_strategy {
    // Automatically backfill objects included in the stream source
    // configuration. Specific objects can be excluded.
    BackfillAllStrategy backfill_all = 101;

    // Do not automatically backfill any objects.
    BackfillNoneStrategy backfill_none = 102;
  }

  // Output only. Errors on the Stream.
  repeated Error errors = 9 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// Represent a user-facing Error.
message Error {
  // A title that explains the reason for the error.
  string reason = 1;

  // A unique identifier for this specific error,
  // allowing it to be traced throughout the system in logs and API responses.
  string error_uuid = 2;

  // A message containing more information about the error that occurred.
  string message = 3;

  // The time when the error occurred.
  google.protobuf.Timestamp error_time = 4;

  // Additional information about the error.
  map<string, string> details = 5;
}

// Contains the current validation results.
message ValidationResult {
  // A list of validations (includes both executed as well as not executed
  // validations).
  repeated Validation validations = 1;
}

message Validation {
  // Validation execution status.
  enum Status {
    // Unspecified status.
    STATUS_UNSPECIFIED = 0;

    // Validation did not execute.
    NOT_EXECUTED = 1;

    // Validation failed.
    FAILED = 2;

    // Validation passed.
    PASSED = 3;
  }

  // A short description of the validation.
  string description = 1;

  // Validation execution status.
  Status status = 2;

  // Messages reflecting the validation results.
  repeated ValidationMessage message = 3;

  // A custom code identifying this validation.
  string code = 4;
}

// Represent user-facing validation result message.
message ValidationMessage {
  // Validation message level.
  enum Level {
    // Unspecified level.
    LEVEL_UNSPECIFIED = 0;

    // Potentially cause issues with the Stream.
    WARNING = 1;

    // Definitely cause issues with the Stream.
    ERROR = 2;
  }

  // The result of the validation.
  string message = 1;

  // Message severity level (warning or error).
  Level level = 2;

  // Additional metadata related to the result.
  map<string, string> metadata = 3;

  // A custom code identifying this specific message.
  string code = 4;
}