aiplatform/v1/pipeline_job.proto

// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package google.cloud.aiplatform.v1;

import "google/api/field_behavior.proto";
import "google/api/resource.proto";
import "google/cloud/aiplatform/v1/artifact.proto";
import "google/cloud/aiplatform/v1/context.proto";
import "google/cloud/aiplatform/v1/encryption_spec.proto";
import "google/cloud/aiplatform/v1/execution.proto";
import "google/cloud/aiplatform/v1/pipeline_failure_policy.proto";
import "google/cloud/aiplatform/v1/pipeline_state.proto";
import "google/cloud/aiplatform/v1/value.proto";
import "google/protobuf/struct.proto";
import "google/protobuf/timestamp.proto";
import "google/rpc/status.proto";

option csharp_namespace = "Google.Cloud.AIPlatform.V1";
option go_package = "cloud.google.com/go/aiplatform/apiv1/aiplatformpb;aiplatformpb";
option java_multiple_files = true;
option java_outer_classname = "Pipeline";
option java_package = "com.google.cloud.aiplatform.v1";
option php_namespace = "Google\\Cloud\\AIPlatform\\V1";
option ruby_package = "Google::Cloud::AIPlatform::V1";
option (google.api.resource_definition) = {
  type: "compute.googleapis.com/Network"
  pattern: "projects/{project}/global/networks/{network}"
};

// An instance of a machine learning PipelineJob.
message PipelineJob {
  option (google.api.resource) = {
    type: "aiplatform.googleapis.com/PipelineJob"
    pattern: "projects/{project}/locations/{location}/pipelineJobs/{pipeline_job}"
  };

  // The runtime config of a PipelineJob.
  message RuntimeConfig {
    // The type of an input artifact.
    message InputArtifact {
      oneof kind {
        // Artifact resource id from MLMD. Which is the last portion of an
        // artifact resource name:
        // `projects/{project}/locations/{location}/metadataStores/default/artifacts/{artifact_id}`.
        // The artifact must stay within the same project, location and default
        // metadatastore as the pipeline.
        string artifact_id = 1;
      }
    }

    // Deprecated. Use
    // [RuntimeConfig.parameter_values][google.cloud.aiplatform.v1.PipelineJob.RuntimeConfig.parameter_values]
    // instead. The runtime parameters of the PipelineJob. The parameters will
    // be passed into
    // [PipelineJob.pipeline_spec][google.cloud.aiplatform.v1.PipelineJob.pipeline_spec]
    // to replace the placeholders at runtime. This field is used by pipelines
    // built using `PipelineJob.pipeline_spec.schema_version` 2.0.0 or lower,
    // such as pipelines built using Kubeflow Pipelines SDK 1.8 or lower.
    map<string, Value> parameters = 1 [deprecated = true];

    // Required. A path in a Cloud Storage bucket, which will be treated as the
    // root output directory of the pipeline. It is used by the system to
    // generate the paths of output artifacts. The artifact paths are generated
    // with a sub-path pattern `{job_id}/{task_id}/{output_key}` under the
    // specified output directory. The service account specified in this
    // pipeline must have the `storage.objects.get` and `storage.objects.create`
    // permissions for this bucket.
    string gcs_output_directory = 2 [(google.api.field_behavior) = REQUIRED];

    // The runtime parameters of the PipelineJob. The parameters will be
    // passed into
    // [PipelineJob.pipeline_spec][google.cloud.aiplatform.v1.PipelineJob.pipeline_spec]
    // to replace the placeholders at runtime. This field is used by pipelines
    // built using `PipelineJob.pipeline_spec.schema_version` 2.1.0, such as
    // pipelines built using Kubeflow Pipelines SDK 1.9 or higher and the v2
    // DSL.
    map<string, google.protobuf.Value> parameter_values = 3;

    // Represents the failure policy of a pipeline. Currently, the default of a
    // pipeline is that the pipeline will continue to run until no more tasks
    // can be executed, also known as PIPELINE_FAILURE_POLICY_FAIL_SLOW.
    // However, if a pipeline is set to PIPELINE_FAILURE_POLICY_FAIL_FAST, it
    // will stop scheduling any new tasks when a task has failed. Any scheduled
    // tasks will continue to completion.
    PipelineFailurePolicy failure_policy = 4;

    // The runtime artifacts of the PipelineJob. The key will be the input
    // artifact name and the value would be one of the InputArtifact.
    map<string, InputArtifact> input_artifacts = 5;
  }

  // Output only. The resource name of the PipelineJob.
  string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

  // The display name of the Pipeline.
  // The name can be up to 128 characters long and can consist of any UTF-8
  // characters.
  string display_name = 2;

  // Output only. Pipeline creation time.
  google.protobuf.Timestamp create_time = 3
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Pipeline start time.
  google.protobuf.Timestamp start_time = 4
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Pipeline end time.
  google.protobuf.Timestamp end_time = 5
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Timestamp when this PipelineJob was most recently updated.
  google.protobuf.Timestamp update_time = 6
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // The spec of the pipeline.
  google.protobuf.Struct pipeline_spec = 7;

  // Output only. The detailed state of the job.
  PipelineState state = 8 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The details of pipeline run. Not available in the list view.
  PipelineJobDetail job_detail = 9 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The error that occurred during pipeline execution.
  // Only populated when the pipeline's state is FAILED or CANCELLED.
  google.rpc.Status error = 10 [(google.api.field_behavior) = OUTPUT_ONLY];

  // The labels with user-defined metadata to organize PipelineJob.
  //
  // Label keys and values can be no longer than 64 characters
  // (Unicode codepoints), can only contain lowercase letters, numeric
  // characters, underscores and dashes. International characters are allowed.
  //
  // See https://goo.gl/xmQnxf for more information and examples of labels.
  //
  // Note there is some reserved label key for Vertex AI Pipelines.
  // - `vertex-ai-pipelines-run-billing-id`, user set value will get overrided.
  map<string, string> labels = 11;

  // Runtime config of the pipeline.
  RuntimeConfig runtime_config = 12;

  // Customer-managed encryption key spec for a pipelineJob. If set, this
  // PipelineJob and all of its sub-resources will be secured by this key.
  EncryptionSpec encryption_spec = 16;

  // The service account that the pipeline workload runs as.
  // If not specified, the Compute Engine default service account in the project
  // will be used.
  // See
  // https://cloud.google.com/compute/docs/access/service-accounts#default_service_account
  //
  // Users starting the pipeline must have the `iam.serviceAccounts.actAs`
  // permission on this service account.
  string service_account = 17;

  // The full name of the Compute Engine
  // [network](/compute/docs/networks-and-firewalls#networks) to which the
  // Pipeline Job's workload should be peered. For example,
  // `projects/12345/global/networks/myVPC`.
  // [Format](/compute/docs/reference/rest/v1/networks/insert)
  // is of the form `projects/{project}/global/networks/{network}`.
  // Where {project} is a project number, as in `12345`, and {network} is a
  // network name.
  //
  // Private services access must already be configured for the network.
  // Pipeline job will apply the network configuration to the Google Cloud
  // resources being launched, if applied, such as Vertex AI
  // Training or Dataflow job. If left unspecified, the workload is not peered
  // with any network.
  string network = 18 [
    (google.api.resource_reference) = { type: "compute.googleapis.com/Network" }
  ];

  // A list of names for the reserved ip ranges under the VPC network
  // that can be used for this Pipeline Job's workload.
  //
  // If set, we will deploy the Pipeline Job's workload within the provided ip
  // ranges. Otherwise, the job will be deployed to any ip ranges under the
  // provided VPC network.
  //
  // Example: ['vertex-ai-ip-range'].
  repeated string reserved_ip_ranges = 25;

  // A template uri from where the
  // [PipelineJob.pipeline_spec][google.cloud.aiplatform.v1.PipelineJob.pipeline_spec],
  // if empty, will be downloaded. Currently, only uri from Vertex Template
  // Registry & Gallery is supported. Reference to
  // https://cloud.google.com/vertex-ai/docs/pipelines/create-pipeline-template.
  string template_uri = 19;

  // Output only. Pipeline template metadata. Will fill up fields if
  // [PipelineJob.template_uri][google.cloud.aiplatform.v1.PipelineJob.template_uri]
  // is from supported template registry.
  PipelineTemplateMetadata template_metadata = 20
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The schedule resource name.
  // Only returned if the Pipeline is created by Schedule API.
  string schedule_name = 22 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// Pipeline template metadata if
// [PipelineJob.template_uri][google.cloud.aiplatform.v1.PipelineJob.template_uri]
// is from supported template registry. Currently, the only supported registry
// is Artifact Registry.
message PipelineTemplateMetadata {
  // The version_name in artifact registry.
  //
  // Will always be presented in output if the
  // [PipelineJob.template_uri][google.cloud.aiplatform.v1.PipelineJob.template_uri]
  // is from supported template registry.
  //
  // Format is "sha256:abcdef123456...".
  string version = 3;
}

// The runtime detail of PipelineJob.
message PipelineJobDetail {
  // Output only. The context of the pipeline.
  Context pipeline_context = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The context of the current pipeline run.
  Context pipeline_run_context = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The runtime details of the tasks under the pipeline.
  repeated PipelineTaskDetail task_details = 3
      [(google.api.field_behavior) = OUTPUT_ONLY];
}

// The runtime detail of a task execution.
message PipelineTaskDetail {
  // A single record of the task status.
  message PipelineTaskStatus {
    // Output only. Update time of this status.
    google.protobuf.Timestamp update_time = 1
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The state of the task.
    State state = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The error that occurred during the state. May be set when
    // the state is any of the non-final state (PENDING/RUNNING/CANCELLING) or
    // FAILED state. If the state is FAILED, the error here is final and not
    // going to be retried. If the state is a non-final state, the error
    // indicates a system-error being retried.
    google.rpc.Status error = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // A list of artifact metadata.
  message ArtifactList {
    // Output only. A list of artifact metadata.
    repeated Artifact artifacts = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // Specifies state of TaskExecution
  enum State {
    // Unspecified.
    STATE_UNSPECIFIED = 0;

    // Specifies pending state for the task.
    PENDING = 1;

    // Specifies task is being executed.
    RUNNING = 2;

    // Specifies task completed successfully.
    SUCCEEDED = 3;

    // Specifies Task cancel is in pending state.
    CANCEL_PENDING = 4;

    // Specifies task is being cancelled.
    CANCELLING = 5;

    // Specifies task was cancelled.
    CANCELLED = 6;

    // Specifies task failed.
    FAILED = 7;

    // Specifies task was skipped due to cache hit.
    SKIPPED = 8;

    // Specifies that the task was not triggered because the task's trigger
    // policy is not satisfied. The trigger policy is specified in the
    // `condition` field of
    // [PipelineJob.pipeline_spec][google.cloud.aiplatform.v1.PipelineJob.pipeline_spec].
    NOT_TRIGGERED = 9;
  }

  // Output only. The system generated ID of the task.
  int64 task_id = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The id of the parent task if the task is within a component
  // scope. Empty if the task is at the root level.
  int64 parent_task_id = 12 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The user specified name of the task that is defined in
  // [pipeline_spec][google.cloud.aiplatform.v1.PipelineJob.pipeline_spec].
  string task_name = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Task create time.
  google.protobuf.Timestamp create_time = 3
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Task start time.
  google.protobuf.Timestamp start_time = 4
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Task end time.
  google.protobuf.Timestamp end_time = 5
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The detailed execution info.
  PipelineTaskExecutorDetail executor_detail = 6
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. State of the task.
  State state = 7 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The execution metadata of the task.
  Execution execution = 8 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The error that occurred during task execution.
  // Only populated when the task's state is FAILED or CANCELLED.
  google.rpc.Status error = 9 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. A list of task status. This field keeps a record of task
  // status evolving over time.
  repeated PipelineTaskStatus pipeline_task_status = 13
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The runtime input artifacts of the task.
  map<string, ArtifactList> inputs = 10
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The runtime output artifacts of the task.
  map<string, ArtifactList> outputs = 11
      [(google.api.field_behavior) = OUTPUT_ONLY];
}

// The runtime detail of a pipeline executor.
message PipelineTaskExecutorDetail {
  // The detail of a container execution. It contains the job names of the
  // lifecycle of a container execution.
  message ContainerDetail {
    // Output only. The name of the
    // [CustomJob][google.cloud.aiplatform.v1.CustomJob] for the main container
    // execution.
    string main_job = 1 [
      (google.api.field_behavior) = OUTPUT_ONLY,
      (google.api.resource_reference) = {
        type: "aiplatform.googleapis.com/CustomJob"
      }
    ];

    // Output only. The name of the
    // [CustomJob][google.cloud.aiplatform.v1.CustomJob] for the
    // pre-caching-check container execution. This job will be available if the
    // [PipelineJob.pipeline_spec][google.cloud.aiplatform.v1.PipelineJob.pipeline_spec]
    // specifies the `pre_caching_check` hook in the lifecycle events.
    string pre_caching_check_job = 2 [
      (google.api.field_behavior) = OUTPUT_ONLY,
      (google.api.resource_reference) = {
        type: "aiplatform.googleapis.com/CustomJob"
      }
    ];

    // Output only. The names of the previously failed
    // [CustomJob][google.cloud.aiplatform.v1.CustomJob] for the main container
    // executions. The list includes the all attempts in chronological order.
    repeated string failed_main_jobs = 3
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The names of the previously failed
    // [CustomJob][google.cloud.aiplatform.v1.CustomJob] for the
    // pre-caching-check container executions. This job will be available if the
    // [PipelineJob.pipeline_spec][google.cloud.aiplatform.v1.PipelineJob.pipeline_spec]
    // specifies the `pre_caching_check` hook in the lifecycle events. The list
    // includes the all attempts in chronological order.
    repeated string failed_pre_caching_check_jobs = 4
        [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // The detailed info for a custom job executor.
  message CustomJobDetail {
    // Output only. The name of the
    // [CustomJob][google.cloud.aiplatform.v1.CustomJob].
    string job = 1 [
      (google.api.field_behavior) = OUTPUT_ONLY,
      (google.api.resource_reference) = {
        type: "aiplatform.googleapis.com/CustomJob"
      }
    ];

    // Output only. The names of the previously failed
    // [CustomJob][google.cloud.aiplatform.v1.CustomJob]. The list includes the
    // all attempts in chronological order.
    repeated string failed_jobs = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  oneof details {
    // Output only. The detailed info for a container executor.
    ContainerDetail container_detail = 1
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The detailed info for a custom job executor.
    CustomJobDetail custom_job_detail = 2
        [(google.api.field_behavior) = OUTPUT_ONLY];
  }
}