dataflow/v1beta3/templates.proto

// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package google.dataflow.v1beta3;

import "google/api/annotations.proto";
import "google/api/client.proto";
import "google/dataflow/v1beta3/environment.proto";
import "google/dataflow/v1beta3/jobs.proto";
import "google/rpc/status.proto";

option csharp_namespace = "Google.Cloud.Dataflow.V1Beta3";
option go_package = "cloud.google.com/go/dataflow/apiv1beta3/dataflowpb;dataflowpb";
option java_multiple_files = true;
option java_outer_classname = "TemplatesProto";
option java_package = "com.google.dataflow.v1beta3";
option php_namespace = "Google\\Cloud\\Dataflow\\V1beta3";
option ruby_package = "Google::Cloud::Dataflow::V1beta3";

// Provides a method to create Cloud Dataflow jobs from templates.
service TemplatesService {
  option (google.api.default_host) = "dataflow.googleapis.com";
  option (google.api.oauth_scopes) =
      "https://www.googleapis.com/auth/cloud-platform,"
      "https://www.googleapis.com/auth/compute,"
      "https://www.googleapis.com/auth/compute.readonly,"
      "https://www.googleapis.com/auth/userinfo.email";

  // Creates a Cloud Dataflow job from a template.
  rpc CreateJobFromTemplate(CreateJobFromTemplateRequest) returns (Job) {
    option (google.api.http) = {
      post: "/v1b3/projects/{project_id}/locations/{location}/templates"
      body: "*"
      additional_bindings {
        post: "/v1b3/projects/{project_id}/templates"
        body: "*"
      }
    };
  }

  // Launch a template.
  rpc LaunchTemplate(LaunchTemplateRequest) returns (LaunchTemplateResponse) {
    option (google.api.http) = {
      post: "/v1b3/projects/{project_id}/locations/{location}/templates:launch"
      body: "launch_parameters"
      additional_bindings {
        post: "/v1b3/projects/{project_id}/templates:launch"
        body: "launch_parameters"
      }
    };
  }

  // Get the template associated with a template.
  rpc GetTemplate(GetTemplateRequest) returns (GetTemplateResponse) {
    option (google.api.http) = {
      get: "/v1b3/projects/{project_id}/locations/{location}/templates:get"
      additional_bindings {
        get: "/v1b3/projects/{project_id}/templates:get"
      }
    };
  }
}

// Provides a service for Flex templates. This feature is not ready yet.
service FlexTemplatesService {
  option (google.api.default_host) = "dataflow.googleapis.com";
  option (google.api.oauth_scopes) =
      "https://www.googleapis.com/auth/cloud-platform,"
      "https://www.googleapis.com/auth/compute,"
      "https://www.googleapis.com/auth/compute.readonly,"
      "https://www.googleapis.com/auth/userinfo.email";

  // Launch a job with a FlexTemplate.
  rpc LaunchFlexTemplate(LaunchFlexTemplateRequest) returns (LaunchFlexTemplateResponse) {
    option (google.api.http) = {
      post: "/v1b3/projects/{project_id}/locations/{location}/flexTemplates:launch"
      body: "*"
    };
  }
}

// Response to the request to launch a job from Flex Template.
message LaunchFlexTemplateResponse {
  // The job that was launched, if the request was not a dry run and
  // the job was successfully launched.
  Job job = 1;
}

// Container Spec.
message ContainerSpec {
  // Name of the docker container image. E.g., gcr.io/project/some-image
  string image = 1;

  // Metadata describing a template including description and validation rules.
  TemplateMetadata metadata = 2;

  // Required. SDK info of the Flex Template.
  SDKInfo sdk_info = 3;

  // Default runtime environment for the job.
  FlexTemplateRuntimeEnvironment default_environment = 4;
}

// Launch FlexTemplate Parameter.
message LaunchFlexTemplateParameter {
  // Required. The job name to use for the created job. For update job request,
  // job name should be same as the existing running job.
  string job_name = 1;

  // Launch Mechanism.
  oneof template {
    // Spec about the container image to launch.
    ContainerSpec container_spec = 4;

    // Cloud Storage path to a file with json serialized ContainerSpec as
    // content.
    string container_spec_gcs_path = 5;
  }

  // The parameters for FlexTemplate.
  // Ex. {"num_workers":"5"}
  map<string, string> parameters = 2;

  // Launch options for this flex template job. This is a common set of options
  // across languages and templates. This should not be used to pass job
  // parameters.
  map<string, string> launch_options = 6;

  // The runtime environment for the FlexTemplate job
  FlexTemplateRuntimeEnvironment environment = 7;

  // Set this to true if you are sending a request to update a running
  // streaming job. When set, the job name should be the same as the
  // running job.
  bool update = 8;

  // Use this to pass transform_name_mappings for streaming update jobs.
  // Ex:{"oldTransformName":"newTransformName",...}'
  map<string, string> transform_name_mappings = 9;
}

// The environment values to be set at runtime for flex template.
message FlexTemplateRuntimeEnvironment {
  // The initial number of Google Compute Engine instances for the job.
  int32 num_workers = 1;

  // The maximum number of Google Compute Engine instances to be made
  // available to your pipeline during execution, from 1 to 1000.
  int32 max_workers = 2;

  // The Compute Engine [availability
  // zone](https://cloud.google.com/compute/docs/regions-zones/regions-zones)
  // for launching worker instances to run your pipeline.
  // In the future, worker_zone will take precedence.
  string zone = 3;

  // The email address of the service account to run the job as.
  string service_account_email = 4;

  // The Cloud Storage path to use for temporary files.
  // Must be a valid Cloud Storage URL, beginning with `gs://`.
  string temp_location = 5;

  // The machine type to use for the job. Defaults to the value from the
  // template if not specified.
  string machine_type = 6;

  // Additional experiment flags for the job.
  repeated string additional_experiments = 7;

  // Network to which VMs will be assigned.  If empty or unspecified,
  // the service will use the network "default".
  string network = 8;

  // Subnetwork to which VMs will be assigned, if desired. You can specify a
  // subnetwork using either a complete URL or an abbreviated path. Expected to
  // be of the form
  // "https://www.googleapis.com/compute/v1/projects/HOST_PROJECT_ID/regions/REGION/subnetworks/SUBNETWORK"
  // or "regions/REGION/subnetworks/SUBNETWORK". If the subnetwork is located in
  // a Shared VPC network, you must use the complete URL.
  string subnetwork = 9;

  // Additional user labels to be specified for the job.
  // Keys and values must follow the restrictions specified in the [labeling
  // restrictions](https://cloud.google.com/compute/docs/labeling-resources#restrictions)
  // page.
  // An object containing a list of "key": value pairs.
  // Example: { "name": "wrench", "mass": "1kg", "count": "3" }.
  map<string, string> additional_user_labels = 10;

  // Name for the Cloud KMS key for the job.
  // Key format is:
  // projects/<project>/locations/<location>/keyRings/<keyring>/cryptoKeys/<key>
  string kms_key_name = 11;

  // Configuration for VM IPs.
  WorkerIPAddressConfiguration ip_configuration = 12;

  // The Compute Engine region
  // (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in
  // which worker processing should occur, e.g. "us-west1". Mutually exclusive
  // with worker_zone. If neither worker_region nor worker_zone is specified,
  // default to the control plane's region.
  string worker_region = 13;

  // The Compute Engine zone
  // (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in
  // which worker processing should occur, e.g. "us-west1-a". Mutually exclusive
  // with worker_region. If neither worker_region nor worker_zone is specified,
  // a zone in the control plane's region is chosen based on available capacity.
  // If both `worker_zone` and `zone` are set, `worker_zone` takes precedence.
  string worker_zone = 14;

  // Whether to enable Streaming Engine for the job.
  bool enable_streaming_engine = 15;

  // Set FlexRS goal for the job.
  // https://cloud.google.com/dataflow/docs/guides/flexrs
  FlexResourceSchedulingGoal flexrs_goal = 16;

  // The Cloud Storage path for staging local files.
  // Must be a valid Cloud Storage URL, beginning with `gs://`.
  string staging_location = 17;

  // Docker registry location of container image to use for the 'worker harness.
  // Default is the container for the version of the SDK. Note this field is
  // only valid for portable pipelines.
  string sdk_container_image = 18;

  // Worker disk size, in gigabytes.
  int32 disk_size_gb = 20;

  // The algorithm to use for autoscaling
  AutoscalingAlgorithm autoscaling_algorithm = 21;

  // If true, save a heap dump before killing a thread or process which is GC
  // thrashing or out of memory. The location of the heap file will either be
  // echoed back to the user, or the user will be given the opportunity to
  // download the heap file.
  bool dump_heap_on_oom = 22;

  // Cloud Storage bucket (directory) to upload heap dumps to the given
  // location. Enabling this implies that heap dumps should be generated on OOM
  // (dump_heap_on_oom is set to true).
  string save_heap_dumps_to_gcs_path = 23;

  // The machine type to use for launching the job. The default is
  // n1-standard-1.
  string launcher_machine_type = 24;
}

// A request to launch a Cloud Dataflow job from a FlexTemplate.
message LaunchFlexTemplateRequest {
  // Required. The ID of the Cloud Platform project that the job belongs to.
  string project_id = 1;

  // Required. Parameter to launch a job form Flex Template.
  LaunchFlexTemplateParameter launch_parameter = 2;

  // Required. The [regional endpoint]
  // (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) to
  // which to direct the request. E.g., us-central1, us-west1.
  string location = 3;

  // If true, the request is validated but not actually executed.
  // Defaults to false.
  bool validate_only = 4;
}

// The environment values to set at runtime.
message RuntimeEnvironment {
  // The initial number of Google Compute Engine instnaces for the job.
  int32 num_workers = 11;

  // The maximum number of Google Compute Engine instances to be made
  // available to your pipeline during execution, from 1 to 1000.
  int32 max_workers = 1;

  // The Compute Engine [availability
  // zone](https://cloud.google.com/compute/docs/regions-zones/regions-zones)
  // for launching worker instances to run your pipeline.
  // In the future, worker_zone will take precedence.
  string zone = 2;

  // The email address of the service account to run the job as.
  string service_account_email = 3;

  // The Cloud Storage path to use for temporary files.
  // Must be a valid Cloud Storage URL, beginning with `gs://`.
  string temp_location = 4;

  // Whether to bypass the safety checks for the job's temporary directory.
  // Use with caution.
  bool bypass_temp_dir_validation = 5;

  // The machine type to use for the job. Defaults to the value from the
  // template if not specified.
  string machine_type = 6;

  // Additional experiment flags for the job, specified with the
  // `--experiments` option.
  repeated string additional_experiments = 7;

  // Network to which VMs will be assigned.  If empty or unspecified,
  // the service will use the network "default".
  string network = 8;

  // Subnetwork to which VMs will be assigned, if desired. You can specify a
  // subnetwork using either a complete URL or an abbreviated path. Expected to
  // be of the form
  // "https://www.googleapis.com/compute/v1/projects/HOST_PROJECT_ID/regions/REGION/subnetworks/SUBNETWORK"
  // or "regions/REGION/subnetworks/SUBNETWORK". If the subnetwork is located in
  // a Shared VPC network, you must use the complete URL.
  string subnetwork = 9;

  // Additional user labels to be specified for the job.
  // Keys and values should follow the restrictions specified in the [labeling
  // restrictions](https://cloud.google.com/compute/docs/labeling-resources#restrictions)
  // page.
  // An object containing a list of "key": value pairs.
  // Example: { "name": "wrench", "mass": "1kg", "count": "3" }.
  map<string, string> additional_user_labels = 10;

  // Name for the Cloud KMS key for the job.
  // Key format is:
  // projects/<project>/locations/<location>/keyRings/<keyring>/cryptoKeys/<key>
  string kms_key_name = 12;

  // Configuration for VM IPs.
  WorkerIPAddressConfiguration ip_configuration = 14;

  // The Compute Engine region
  // (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in
  // which worker processing should occur, e.g. "us-west1". Mutually exclusive
  // with worker_zone. If neither worker_region nor worker_zone is specified,
  // default to the control plane's region.
  string worker_region = 15;

  // The Compute Engine zone
  // (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in
  // which worker processing should occur, e.g. "us-west1-a". Mutually exclusive
  // with worker_region. If neither worker_region nor worker_zone is specified,
  // a zone in the control plane's region is chosen based on available capacity.
  // If both `worker_zone` and `zone` are set, `worker_zone` takes precedence.
  string worker_zone = 16;

  // Whether to enable Streaming Engine for the job.
  bool enable_streaming_engine = 17;
}

// Metadata for a specific parameter.
message ParameterMetadata {
  // Required. The name of the parameter.
  string name = 1;

  // Required. The label to display for the parameter.
  string label = 2;

  // Required. The help text to display for the parameter.
  string help_text = 3;

  // Optional. Whether the parameter is optional. Defaults to false.
  bool is_optional = 4;

  // Optional. Regexes that the parameter must match.
  repeated string regexes = 5;

  // Optional. The type of the parameter.
  // Used for selecting input picker.
  ParameterType param_type = 6;

  // Optional. Additional metadata for describing this parameter.
  map<string, string> custom_metadata = 7;
}

// ParameterType specifies what kind of input we need for this parameter.
enum ParameterType {
  // Default input type.
  DEFAULT = 0;

  // The parameter specifies generic text input.
  TEXT = 1;

  // The parameter specifies a Cloud Storage Bucket to read from.
  GCS_READ_BUCKET = 2;

  // The parameter specifies a Cloud Storage Bucket to write to.
  GCS_WRITE_BUCKET = 3;

  // The parameter specifies a Cloud Storage file path to read from.
  GCS_READ_FILE = 4;

  // The parameter specifies a Cloud Storage file path to write to.
  GCS_WRITE_FILE = 5;

  // The parameter specifies a Cloud Storage folder path to read from.
  GCS_READ_FOLDER = 6;

  // The parameter specifies a Cloud Storage folder to write to.
  GCS_WRITE_FOLDER = 7;

  // The parameter specifies a Pub/Sub Topic.
  PUBSUB_TOPIC = 8;

  // The parameter specifies a Pub/Sub Subscription.
  PUBSUB_SUBSCRIPTION = 9;
}

// Metadata describing a template.
message TemplateMetadata {
  // Required. The name of the template.
  string name = 1;

  // Optional. A description of the template.
  string description = 2;

  // The parameters for the template.
  repeated ParameterMetadata parameters = 3;
}

// SDK Information.
message SDKInfo {
  // SDK Language.
  enum Language {
    // UNKNOWN Language.
    UNKNOWN = 0;

    // Java.
    JAVA = 1;

    // Python.
    PYTHON = 2;
  }

  // Required. The SDK Language.
  Language language = 1;

  // Optional. The SDK version.
  string version = 2;
}

// RuntimeMetadata describing a runtime environment.
message RuntimeMetadata {
  // SDK Info for the template.
  SDKInfo sdk_info = 1;

  // The parameters for the template.
  repeated ParameterMetadata parameters = 2;
}

// A request to create a Cloud Dataflow job from a template.
message CreateJobFromTemplateRequest {
  // Required. The ID of the Cloud Platform project that the job belongs to.
  string project_id = 1;

  // Required. The job name to use for the created job.
  string job_name = 4;

  // The template from which to create the job.
  oneof template {
    // Required. A Cloud Storage path to the template from which to
    // create the job.
    // Must be a valid Cloud Storage URL, beginning with `gs://`.
    string gcs_path = 2;
  }

  // The runtime parameters to pass to the job.
  map<string, string> parameters = 3;

  // The runtime environment for the job.
  RuntimeEnvironment environment = 5;

  // The [regional endpoint]
  // (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) to
  // which to direct the request.
  string location = 6;
}

// A request to retrieve a Cloud Dataflow job template.
message GetTemplateRequest {
  // The various views of a template that may be retrieved.
  enum TemplateView {
    // Template view that retrieves only the metadata associated with the
    // template.
    METADATA_ONLY = 0;
  }

  // Required. The ID of the Cloud Platform project that the job belongs to.
  string project_id = 1;

  // The template from which to create the job.
  oneof template {
    // Required. A Cloud Storage path to the template from which to
    // create the job.
    // Must be valid Cloud Storage URL, beginning with 'gs://'.
    string gcs_path = 2;
  }

  // The view to retrieve. Defaults to METADATA_ONLY.
  TemplateView view = 3;

  // The [regional endpoint]
  // (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) to
  // which to direct the request.
  string location = 4;
}

// The response to a GetTemplate request.
message GetTemplateResponse {
  // Template Type.
  enum TemplateType {
    // Unknown Template Type.
    UNKNOWN = 0;

    // Legacy Template.
    LEGACY = 1;

    // Flex Template.
    FLEX = 2;
  }

  // The status of the get template request. Any problems with the
  // request will be indicated in the error_details.
  google.rpc.Status status = 1;

  // The template metadata describing the template name, available
  // parameters, etc.
  TemplateMetadata metadata = 2;

  // Template Type.
  TemplateType template_type = 3;

  // Describes the runtime metadata with SDKInfo and available parameters.
  RuntimeMetadata runtime_metadata = 4;
}

// Parameters to provide to the template being launched.
message LaunchTemplateParameters {
  // Required. The job name to use for the created job.
  string job_name = 1;

  // The runtime parameters to pass to the job.
  map<string, string> parameters = 2;

  // The runtime environment for the job.
  RuntimeEnvironment environment = 3;

  // If set, replace the existing pipeline with the name specified by jobName
  // with this pipeline, preserving state.
  bool update = 4;

  // Only applicable when updating a pipeline. Map of transform name prefixes of
  // the job to be replaced to the corresponding name prefixes of the new job.
  map<string, string> transform_name_mapping = 5;
}

// A request to launch a template.
message LaunchTemplateRequest {
  // Required. The ID of the Cloud Platform project that the job belongs to.
  string project_id = 1;

  // If true, the request is validated but not actually executed.
  // Defaults to false.
  bool validate_only = 2;

  // The template from which to create the job.
  oneof template {
    // A Cloud Storage path to the template from which to create
    // the job.
    // Must be valid Cloud Storage URL, beginning with 'gs://'.
    string gcs_path = 3;

    // Params for launching a dynamic template.
    DynamicTemplateLaunchParams dynamic_template = 6;
  }

  // The parameters of the template to launch. This should be part of the
  // body of the POST request.
  LaunchTemplateParameters launch_parameters = 4;

  // The [regional endpoint]
  // (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) to
  // which to direct the request.
  string location = 5;
}

// Response to the request to launch a template.
message LaunchTemplateResponse {
  // The job that was launched, if the request was not a dry run and
  // the job was successfully launched.
  Job job = 1;
}

// Used in the error_details field of a google.rpc.Status message, this
// indicates problems with the template parameter.
message InvalidTemplateParameters {
  // A specific template-parameter violation.
  message ParameterViolation {
    // The parameter that failed to validate.
    string parameter = 1;

    // A description of why the parameter failed to validate.
    string description = 2;
  }

  // Describes all parameter violations in a template request.
  repeated ParameterViolation parameter_violations = 1;
}

// Params which should be passed when launching a dynamic template.
message DynamicTemplateLaunchParams {
  // Path to dynamic template spec file on Cloud Storage.
  // The file must be a Json serialized DynamicTemplateFieSpec object.
  string gcs_path = 1;

  // Cloud Storage path for staging dependencies.
  // Must be a valid Cloud Storage URL, beginning with `gs://`.
  string staging_location = 2;
}