datafusion/v1/datafusion.proto

// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package google.cloud.datafusion.v1;

import "google/api/annotations.proto";
import "google/api/client.proto";
import "google/api/field_behavior.proto";
import "google/api/resource.proto";
import "google/longrunning/operations.proto";
import "google/protobuf/field_mask.proto";
import "google/protobuf/timestamp.proto";

option csharp_namespace = "Google.Cloud.DataFusion.V1";
option go_package = "cloud.google.com/go/datafusion/apiv1/datafusionpb;datafusionpb";
option java_multiple_files = true;
option java_package = "com.google.cloud.datafusion.v1";
option php_namespace = "Google\\Cloud\\DataFusion\\V1";
option ruby_package = "Google::Cloud::DataFusion::V1";
option (google.api.resource_definition) = {
  type: "cloudkms.googleapis.com/CryptoKey"
  pattern: "projects/{project}/locations/{location}/keyRings/{key_ring}/cryptoKeys/{crypto_key}"
};

// Service for creating and managing Data Fusion instances.
// Data Fusion enables ETL developers to build code-free, data integration
// pipelines via a point-and-click UI.
service DataFusion {
  option (google.api.default_host) = "datafusion.googleapis.com";
  option (google.api.oauth_scopes) = "https://www.googleapis.com/auth/cloud-platform";

  // Lists possible versions for Data Fusion instances in the specified project
  // and location.
  rpc ListAvailableVersions(ListAvailableVersionsRequest) returns (ListAvailableVersionsResponse) {
    option (google.api.http) = {
      get: "/v1/{parent=projects/*/locations/*}/versions"
    };
    option (google.api.method_signature) = "parent";
  }

  // Lists Data Fusion instances in the specified project and location.
  rpc ListInstances(ListInstancesRequest) returns (ListInstancesResponse) {
    option (google.api.http) = {
      get: "/v1/{parent=projects/*/locations/*}/instances"
    };
  }

  // Gets details of a single Data Fusion instance.
  rpc GetInstance(GetInstanceRequest) returns (Instance) {
    option (google.api.http) = {
      get: "/v1/{name=projects/*/locations/*/instances/*}"
    };
  }

  // Creates a new Data Fusion instance in the specified project and location.
  rpc CreateInstance(CreateInstanceRequest) returns (google.longrunning.Operation) {
    option (google.api.http) = {
      post: "/v1/{parent=projects/*/locations/*}/instances"
      body: "instance"
    };
    option (google.api.method_signature) = "parent,instance,instance_id";
    option (google.longrunning.operation_info) = {
      response_type: "Instance"
      metadata_type: "OperationMetadata"
    };
  }

  // Deletes a single Date Fusion instance.
  rpc DeleteInstance(DeleteInstanceRequest) returns (google.longrunning.Operation) {
    option (google.api.http) = {
      delete: "/v1/{name=projects/*/locations/*/instances/*}"
    };
    option (google.api.method_signature) = "name";
    option (google.longrunning.operation_info) = {
      response_type: "google.protobuf.Empty"
      metadata_type: "OperationMetadata"
    };
  }

  // Updates a single Data Fusion instance.
  rpc UpdateInstance(UpdateInstanceRequest) returns (google.longrunning.Operation) {
    option (google.api.http) = {
      patch: "/v1/{instance.name=projects/*/locations/*/instances/*}"
      body: "instance"
    };
    option (google.api.method_signature) = "instance,update_mask";
    option (google.longrunning.operation_info) = {
      response_type: "Instance"
      metadata_type: "OperationMetadata"
    };
  }

  // Restart a single Data Fusion instance.
  // At the end of an operation instance is fully restarted.
  rpc RestartInstance(RestartInstanceRequest) returns (google.longrunning.Operation) {
    option (google.api.http) = {
      post: "/v1/{name=projects/*/locations/*/instances/*}:restart"
      body: "*"
    };
    option (google.longrunning.operation_info) = {
      response_type: "Instance"
      metadata_type: "OperationMetadata"
    };
  }
}

// Network configuration for a Data Fusion instance. These configurations
// are used for peering with the customer network. Configurations are optional
// when a public Data Fusion instance is to be created. However, providing
// these configurations allows several benefits, such as reduced network latency
// while accessing the customer resources from managed Data Fusion instance
// nodes, as well as access to the customer on-prem resources.
message NetworkConfig {
  // Name of the network in the customer project with which the Tenant Project
  // will be peered for executing pipelines. In case of shared VPC where the
  // network resides in another host project the network should specified in
  // the form of projects/{host-project-id}/global/networks/{network}
  string network = 1;

  // The IP range in CIDR notation to use for the managed Data Fusion instance
  // nodes. This range must not overlap with any other ranges used in the
  // customer network.
  string ip_allocation = 2;
}

// The Data Fusion version. This proto message stores information about certain
// Data Fusion version, which is used for Data Fusion version upgrade.
message Version {
  // Each type represents the release availability of a CDF version
  enum Type {
    // Version does not have availability yet
    TYPE_UNSPECIFIED = 0;

    // Version is under development and not considered stable
    TYPE_PREVIEW = 1;

    // Version is available for public use
    TYPE_GENERAL_AVAILABILITY = 2;
  }

  // The version number of the Data Fusion instance, such as '6.0.1.0'.
  string version_number = 1;

  // Whether this is currently the default version for Cloud Data Fusion
  bool default_version = 2;

  // Represents a list of available feature names for a given version.
  repeated string available_features = 3;

  // Type represents the release availability of the version
  Type type = 4;
}

// Identifies Data Fusion accelerators for an instance.
message Accelerator {
  // Each type represents an Accelerator (Add-On) supported by Cloud Data Fusion
  // service.
  enum AcceleratorType {
    // Default value, if unspecified.
    ACCELERATOR_TYPE_UNSPECIFIED = 0;

    // Change Data Capture accelerator for CDF.
    CDC = 1;

    // Cloud Healthcare accelerator for CDF. This accelerator is to enable Cloud
    // Healthcare specific CDF plugins developed by Healthcare team.
    HEALTHCARE = 2;

    // Contact Center AI Insights
    // This accelerator is used to enable import and export pipelines
    // custom built to streamline CCAI Insights processing.
    CCAI_INSIGHTS = 3;
  }

  // Different values possible for the state of an accelerator
  enum State {
    // Default value, do not use
    STATE_UNSPECIFIED = 0;

    // Indicates that the accelerator is enabled and available to use
    ENABLED = 1;

    // Indicates that the accelerator is disabled and not available to use
    DISABLED = 2;

    // Indicates that accelerator state is currently unknown.
    // Requests for enable, disable could be retried while in this state
    UNKNOWN = 3;
  }

  // The type of an accelator for a CDF instance.
  AcceleratorType accelerator_type = 1;

  // The state of the accelerator
  State state = 2;
}

// The crypto key configuration. This field is used by the Customer-managed
// encryption keys (CMEK) feature.
message CryptoKeyConfig {
  // The name of the key which is used to encrypt/decrypt customer data. For key
  // in Cloud KMS, the key should be in the format of
  // `projects/*/locations/*/keyRings/*/cryptoKeys/*`.
  string key_reference = 1 [(google.api.resource_reference) = {
                              type: "cloudkms.googleapis.com/CryptoKey"
                            }];
}

// Represents a Data Fusion instance.
message Instance {
  option (google.api.resource) = {
    type: "datafusion.googleapis.com/Instance"
    pattern: "projects/{project}/locations/{location}/instances/{instance}"
  };

  // Represents the type of Data Fusion instance. Each type is configured with
  // the default settings for processing and memory.
  enum Type {
    // No type specified. The instance creation will fail.
    TYPE_UNSPECIFIED = 0;

    // Basic Data Fusion instance. In Basic type, the user will be able to
    // create data pipelines using point and click UI. However, there are
    // certain limitations, such as fewer number of concurrent pipelines, no
    // support for streaming pipelines, etc.
    BASIC = 1;

    // Enterprise Data Fusion instance. In Enterprise type, the user will have
    // all features available, such as support for streaming pipelines, higher
    // number of concurrent pipelines, etc.
    ENTERPRISE = 2;

    // Developer Data Fusion instance. In Developer type, the user will have all
    // features available but with restrictive capabilities. This is to help
    // enterprises design and develop their data ingestion and integration
    // pipelines at low cost.
    DEVELOPER = 3;
  }

  // Represents the state of a Data Fusion instance
  enum State {
    // Instance does not have a state yet
    STATE_UNSPECIFIED = 0;

    // Instance is being created
    CREATING = 1;

    // Instance is active and ready for requests. This corresponds to 'RUNNING'
    // in datafusion.v1beta1.
    ACTIVE = 2;

    // Instance creation failed
    FAILED = 3;

    // Instance is being deleted
    DELETING = 4;

    // Instance is being upgraded
    UPGRADING = 5;

    // Instance is being restarted
    RESTARTING = 6;

    // Instance is being updated on customer request
    UPDATING = 7;

    // Instance is being auto-updated
    AUTO_UPDATING = 8;

    // Instance is being auto-upgraded
    AUTO_UPGRADING = 9;

    // Instance is disabled
    DISABLED = 10;
  }

  // The reason for disabling the instance if the state is DISABLED.
  enum DisabledReason {
    // This is an unknown reason for disabling.
    DISABLED_REASON_UNSPECIFIED = 0;

    // The KMS key used by the instance is either revoked or denied access to
    KMS_KEY_ISSUE = 1;
  }

  // Output only. The name of this instance is in the form of
  // projects/{project}/locations/{location}/instances/{instance}.
  string name = 1 [
    (google.api.field_behavior) = OUTPUT_ONLY,
    (google.api.resource_reference) = {
      type: "datafusion.googleapis.com/Instance"
    }
  ];

  // A description of this instance.
  string description = 2;

  // Required. Instance type.
  Type type = 3 [(google.api.field_behavior) = REQUIRED];

  // Option to enable Stackdriver Logging.
  bool enable_stackdriver_logging = 4;

  // Option to enable Stackdriver Monitoring.
  bool enable_stackdriver_monitoring = 5;

  // Specifies whether the Data Fusion instance should be private. If set to
  // true, all Data Fusion nodes will have private IP addresses and will not be
  // able to access the public internet.
  bool private_instance = 6;

  // Network configuration options. These are required when a private Data
  // Fusion instance is to be created.
  NetworkConfig network_config = 7;

  // The resource labels for instance to use to annotate any related underlying
  // resources such as Compute Engine VMs. The character '=' is not allowed to
  // be used within the labels.
  map<string, string> labels = 8;

  // Map of additional options used to configure the behavior of
  // Data Fusion instance.
  map<string, string> options = 9;

  // Output only. The time the instance was created.
  google.protobuf.Timestamp create_time = 10 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The time the instance was last updated.
  google.protobuf.Timestamp update_time = 11 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The current state of this Data Fusion instance.
  State state = 12 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Additional information about the current state of this Data
  // Fusion instance if available.
  string state_message = 13 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Endpoint on which the Data Fusion UI is accessible.
  string service_endpoint = 14 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Name of the zone in which the Data Fusion instance will be created. Only
  // DEVELOPER instances use this field.
  string zone = 15;

  // Current version of the Data Fusion. Only specifiable in Update.
  string version = 16;

  // Output only. Deprecated. Use tenant_project_id instead to extract the tenant project ID.
  string service_account = 17 [
    deprecated = true,
    (google.api.field_behavior) = OUTPUT_ONLY
  ];

  // Display name for an instance.
  string display_name = 18;

  // Available versions that the instance can be upgraded to using
  // UpdateInstanceRequest.
  repeated Version available_version = 19;

  // Output only. Endpoint on which the REST APIs is accessible.
  string api_endpoint = 20 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Cloud Storage bucket generated by Data Fusion in the customer project.
  string gcs_bucket = 21 [(google.api.field_behavior) = OUTPUT_ONLY];

  // List of accelerators enabled for this CDF instance.
  repeated Accelerator accelerators = 22;

  // Output only. P4 service account for the customer project.
  string p4_service_account = 23 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The name of the tenant project.
  string tenant_project_id = 24 [(google.api.field_behavior) = OUTPUT_ONLY];

  // User-managed service account to set on Dataproc when Cloud Data Fusion
  // creates Dataproc to run data processing pipelines.
  //
  // This allows users to have fine-grained access control on Dataproc's
  // accesses to cloud resources.
  string dataproc_service_account = 25;

  // Option to enable granular role-based access control.
  bool enable_rbac = 27;

  // The crypto key configuration. This field is used by the Customer-Managed
  // Encryption Keys (CMEK) feature.
  CryptoKeyConfig crypto_key_config = 28;

  // Output only. If the instance state is DISABLED, the reason for disabling the instance.
  repeated DisabledReason disabled_reason = 29 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// Request message for listing Data Fusion instances.
message ListInstancesRequest {
  // Required. The project and location for which to retrieve instance information
  // in the format projects/{project}/locations/{location}. If the location is
  // specified as '-' (wildcard), then all regions available to the project
  // are queried, and the results are aggregated.
  string parent = 1 [
    (google.api.field_behavior) = REQUIRED,
    (google.api.resource_reference) = {
      type: "locations.googleapis.com/Location"
    }
  ];

  // The maximum number of items to return.
  int32 page_size = 2;

  // The next_page_token value to use if there are additional
  // results to retrieve for this list request.
  string page_token = 3;

  // List filter.
  string filter = 4;

  // Sort results. Supported values are "name", "name desc",  or "" (unsorted).
  string order_by = 5;
}

// Response message for the list instance request.
message ListInstancesResponse {
  // Represents a list of Data Fusion instances.
  repeated Instance instances = 1;

  // Token to retrieve the next page of results or empty if there are no more
  // results in the list.
  string next_page_token = 2;

  // Locations that could not be reached.
  repeated string unreachable = 3;
}

// Request message for the list available versions request.
message ListAvailableVersionsRequest {
  // Required. The project and location for which to retrieve instance information
  // in the format projects/{project}/locations/{location}.
  string parent = 1 [
    (google.api.field_behavior) = REQUIRED,
    (google.api.resource_reference) = {
      type: "locations.googleapis.com/Location"
    }
  ];

  // The maximum number of items to return.
  int32 page_size = 2;

  // The next_page_token value to use if there are additional
  // results to retrieve for this list request.
  string page_token = 3;

  // Whether or not to return the latest patch of every available minor version.
  // If true, only the latest patch will be returned. Ex. if allowed versions is
  // [6.1.1, 6.1.2, 6.2.0] then response will be [6.1.2, 6.2.0]
  bool latest_patch_only = 4;
}

// Response message for the list available versions request.
message ListAvailableVersionsResponse {
  // Represents a list of versions that are supported.
  repeated Version available_versions = 1;

  // Token to retrieve the next page of results or empty if there are no more
  // results in the list.
  string next_page_token = 2;
}

// Request message for getting details about a Data Fusion instance.
message GetInstanceRequest {
  // Required. The instance resource name in the format
  // projects/{project}/locations/{location}/instances/{instance}.
  string name = 1 [
    (google.api.field_behavior) = REQUIRED,
    (google.api.resource_reference) = {
      type: "datafusion.googleapis.com/Instance"
    }
  ];
}

// Request message for creating a Data Fusion instance.
message CreateInstanceRequest {
  // Required. The instance's project and location in the format
  // projects/{project}/locations/{location}.
  string parent = 1 [
    (google.api.field_behavior) = REQUIRED,
    (google.api.resource_reference) = {
      type: "locations.googleapis.com/Location"
    }
  ];

  // Required. The name of the instance to create.
  string instance_id = 2 [(google.api.field_behavior) = REQUIRED];

  // An instance resource.
  Instance instance = 3;
}

// Request message for deleting a Data Fusion instance.
message DeleteInstanceRequest {
  // Required. The instance resource name in the format
  // projects/{project}/locations/{location}/instances/{instance}
  string name = 1 [
    (google.api.field_behavior) = REQUIRED,
    (google.api.resource_reference) = {
      type: "datafusion.googleapis.com/Instance"
    }
  ];
}

// Request message for updating a Data Fusion instance.
// Data Fusion allows updating the labels, options, and stack driver settings.
// This is also used for CDF version upgrade.
message UpdateInstanceRequest {
  // Required. The instance resource that replaces the resource on the server. Currently,
  // Data Fusion only allows replacing labels, options, and stack driver
  // settings. All other fields will be ignored.
  Instance instance = 1 [(google.api.field_behavior) = REQUIRED];

  // Field mask is used to specify the fields that the update will overwrite
  // in an instance resource. The fields specified in the update_mask are
  // relative to the resource, not the full request.
  // A field will be overwritten if it is in the mask.
  // If the user does not provide a mask, all the supported fields (labels,
  // options, and version currently) will be overwritten.
  google.protobuf.FieldMask update_mask = 2;
}

// Request message for restarting a Data Fusion instance.
message RestartInstanceRequest {
  // Required. Name of the Data Fusion instance which need to be restarted in the form of
  // projects/{project}/locations/{location}/instances/{instance}
  string name = 1 [
    (google.api.field_behavior) = REQUIRED,
    (google.api.resource_reference) = {
      type: "datafusion.googleapis.com/Instance"
    }
  ];
}

// Represents the metadata of a long-running operation.
message OperationMetadata {
  // The time the operation was created.
  google.protobuf.Timestamp create_time = 1;

  // The time the operation finished running.
  google.protobuf.Timestamp end_time = 2;

  // Server-defined resource path for the target of the operation.
  string target = 3;

  // Name of the verb executed by the operation.
  string verb = 4;

  // Human-readable status of the operation if any.
  string status_detail = 5;

  // Identifies whether the user has requested cancellation
  // of the operation. Operations that have successfully been cancelled
  // have [Operation.error][] value with a [google.rpc.Status.code][google.rpc.Status.code] of 1,
  // corresponding to `Code.CANCELLED`.
  bool requested_cancellation = 6;

  // API version used to start the operation.
  string api_version = 7;

  // Map to hold any additional status info for the operation
  // If there is an accelerator being enabled/disabled/deleted, this will be
  // populated with accelerator name as key and status as
  // ENABLING, DISABLING or DELETING
  map<string, string> additional_status = 8;
}