aiplatform/v1beta1/index_service.proto

// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package google.cloud.aiplatform.v1beta1;

import "google/api/annotations.proto";
import "google/api/client.proto";
import "google/api/field_behavior.proto";
import "google/api/resource.proto";
import "google/cloud/aiplatform/v1beta1/index.proto";
import "google/cloud/aiplatform/v1beta1/operation.proto";
import "google/longrunning/operations.proto";
import "google/protobuf/empty.proto";
import "google/protobuf/field_mask.proto";

option csharp_namespace = "Google.Cloud.AIPlatform.V1Beta1";
option go_package = "cloud.google.com/go/aiplatform/apiv1beta1/aiplatformpb;aiplatformpb";
option java_multiple_files = true;
option java_outer_classname = "IndexServiceProto";
option java_package = "com.google.cloud.aiplatform.v1beta1";
option php_namespace = "Google\\Cloud\\AIPlatform\\V1beta1";
option ruby_package = "Google::Cloud::AIPlatform::V1beta1";

// A service for creating and managing Vertex AI's Index resources.
service IndexService {
  option (google.api.default_host) = "aiplatform.googleapis.com";
  option (google.api.oauth_scopes) =
      "https://www.googleapis.com/auth/cloud-platform";

  // Creates an Index.
  rpc CreateIndex(CreateIndexRequest) returns (google.longrunning.Operation) {
    option (google.api.http) = {
      post: "/v1beta1/{parent=projects/*/locations/*}/indexes"
      body: "index"
    };
    option (google.api.method_signature) = "parent,index";
    option (google.longrunning.operation_info) = {
      response_type: "Index"
      metadata_type: "CreateIndexOperationMetadata"
    };
  }

  // Gets an Index.
  rpc GetIndex(GetIndexRequest) returns (Index) {
    option (google.api.http) = {
      get: "/v1beta1/{name=projects/*/locations/*/indexes/*}"
    };
    option (google.api.method_signature) = "name";
  }

  // Lists Indexes in a Location.
  rpc ListIndexes(ListIndexesRequest) returns (ListIndexesResponse) {
    option (google.api.http) = {
      get: "/v1beta1/{parent=projects/*/locations/*}/indexes"
    };
    option (google.api.method_signature) = "parent";
  }

  // Updates an Index.
  rpc UpdateIndex(UpdateIndexRequest) returns (google.longrunning.Operation) {
    option (google.api.http) = {
      patch: "/v1beta1/{index.name=projects/*/locations/*/indexes/*}"
      body: "index"
    };
    option (google.api.method_signature) = "index,update_mask";
    option (google.longrunning.operation_info) = {
      response_type: "Index"
      metadata_type: "UpdateIndexOperationMetadata"
    };
  }

  // Deletes an Index.
  // An Index can only be deleted when all its
  // [DeployedIndexes][google.cloud.aiplatform.v1beta1.Index.deployed_indexes]
  // had been undeployed.
  rpc DeleteIndex(DeleteIndexRequest) returns (google.longrunning.Operation) {
    option (google.api.http) = {
      delete: "/v1beta1/{name=projects/*/locations/*/indexes/*}"
    };
    option (google.api.method_signature) = "name";
    option (google.longrunning.operation_info) = {
      response_type: "google.protobuf.Empty"
      metadata_type: "DeleteOperationMetadata"
    };
  }

  // Add/update Datapoints into an Index.
  rpc UpsertDatapoints(UpsertDatapointsRequest)
      returns (UpsertDatapointsResponse) {
    option (google.api.http) = {
      post: "/v1beta1/{index=projects/*/locations/*/indexes/*}:upsertDatapoints"
      body: "*"
    };
  }

  // Remove Datapoints from an Index.
  rpc RemoveDatapoints(RemoveDatapointsRequest)
      returns (RemoveDatapointsResponse) {
    option (google.api.http) = {
      post: "/v1beta1/{index=projects/*/locations/*/indexes/*}:removeDatapoints"
      body: "*"
    };
  }
}

// Request message for
// [IndexService.CreateIndex][google.cloud.aiplatform.v1beta1.IndexService.CreateIndex].
message CreateIndexRequest {
  // Required. The resource name of the Location to create the Index in.
  // Format: `projects/{project}/locations/{location}`
  string parent = 1 [
    (google.api.field_behavior) = REQUIRED,
    (google.api.resource_reference) = {
      type: "locations.googleapis.com/Location"
    }
  ];

  // Required. The Index to create.
  Index index = 2 [(google.api.field_behavior) = REQUIRED];
}

// Runtime operation information for
// [IndexService.CreateIndex][google.cloud.aiplatform.v1beta1.IndexService.CreateIndex].
message CreateIndexOperationMetadata {
  // The operation generic information.
  GenericOperationMetadata generic_metadata = 1;

  // The operation metadata with regard to Matching Engine Index operation.
  NearestNeighborSearchOperationMetadata
      nearest_neighbor_search_operation_metadata = 2;
}

// Request message for
// [IndexService.GetIndex][google.cloud.aiplatform.v1beta1.IndexService.GetIndex]
message GetIndexRequest {
  // Required. The name of the Index resource.
  // Format:
  // `projects/{project}/locations/{location}/indexes/{index}`
  string name = 1 [
    (google.api.field_behavior) = REQUIRED,
    (google.api.resource_reference) = {
      type: "aiplatform.googleapis.com/Index"
    }
  ];
}

// Request message for
// [IndexService.ListIndexes][google.cloud.aiplatform.v1beta1.IndexService.ListIndexes].
message ListIndexesRequest {
  // Required. The resource name of the Location from which to list the Indexes.
  // Format: `projects/{project}/locations/{location}`
  string parent = 1 [
    (google.api.field_behavior) = REQUIRED,
    (google.api.resource_reference) = {
      type: "locations.googleapis.com/Location"
    }
  ];

  // The standard list filter.
  string filter = 2;

  // The standard list page size.
  int32 page_size = 3;

  // The standard list page token.
  // Typically obtained via
  // [ListIndexesResponse.next_page_token][google.cloud.aiplatform.v1beta1.ListIndexesResponse.next_page_token]
  // of the previous
  // [IndexService.ListIndexes][google.cloud.aiplatform.v1beta1.IndexService.ListIndexes]
  // call.
  string page_token = 4;

  // Mask specifying which fields to read.
  google.protobuf.FieldMask read_mask = 5;
}

// Response message for
// [IndexService.ListIndexes][google.cloud.aiplatform.v1beta1.IndexService.ListIndexes].
message ListIndexesResponse {
  // List of indexes in the requested page.
  repeated Index indexes = 1;

  // A token to retrieve next page of results.
  // Pass to
  // [ListIndexesRequest.page_token][google.cloud.aiplatform.v1beta1.ListIndexesRequest.page_token]
  // to obtain that page.
  string next_page_token = 2;
}

// Request message for
// [IndexService.UpdateIndex][google.cloud.aiplatform.v1beta1.IndexService.UpdateIndex].
message UpdateIndexRequest {
  // Required. The Index which updates the resource on the server.
  Index index = 1 [(google.api.field_behavior) = REQUIRED];

  // The update mask applies to the resource.
  // For the `FieldMask` definition, see
  // [google.protobuf.FieldMask][google.protobuf.FieldMask].
  google.protobuf.FieldMask update_mask = 2;
}

// Runtime operation information for
// [IndexService.UpdateIndex][google.cloud.aiplatform.v1beta1.IndexService.UpdateIndex].
message UpdateIndexOperationMetadata {
  // The operation generic information.
  GenericOperationMetadata generic_metadata = 1;

  // The operation metadata with regard to Matching Engine Index operation.
  NearestNeighborSearchOperationMetadata
      nearest_neighbor_search_operation_metadata = 2;
}

// Request message for
// [IndexService.DeleteIndex][google.cloud.aiplatform.v1beta1.IndexService.DeleteIndex].
message DeleteIndexRequest {
  // Required. The name of the Index resource to be deleted.
  // Format:
  // `projects/{project}/locations/{location}/indexes/{index}`
  string name = 1 [
    (google.api.field_behavior) = REQUIRED,
    (google.api.resource_reference) = {
      type: "aiplatform.googleapis.com/Index"
    }
  ];
}

// Request message for
// [IndexService.UpsertDatapoints][google.cloud.aiplatform.v1beta1.IndexService.UpsertDatapoints]
message UpsertDatapointsRequest {
  // Required. The name of the Index resource to be updated.
  // Format:
  // `projects/{project}/locations/{location}/indexes/{index}`
  string index = 1 [
    (google.api.field_behavior) = REQUIRED,
    (google.api.resource_reference) = {
      type: "aiplatform.googleapis.com/Index"
    }
  ];

  // A list of datapoints to be created/updated.
  repeated IndexDatapoint datapoints = 2;

  // Optional. Update mask is used to specify the fields to be overwritten in
  // the datapoints by the update. The fields specified in the update_mask are
  // relative to each IndexDatapoint inside datapoints, not the full request.
  //
  // Updatable fields:
  //
  //   * Use `all_restricts` to update both restricts and numeric_restricts.
  google.protobuf.FieldMask update_mask = 3
      [(google.api.field_behavior) = OPTIONAL];
}

// Response message for
// [IndexService.UpsertDatapoints][google.cloud.aiplatform.v1beta1.IndexService.UpsertDatapoints]
message UpsertDatapointsResponse {}

// Request message for
// [IndexService.RemoveDatapoints][google.cloud.aiplatform.v1beta1.IndexService.RemoveDatapoints]
message RemoveDatapointsRequest {
  // Required. The name of the Index resource to be updated.
  // Format:
  // `projects/{project}/locations/{location}/indexes/{index}`
  string index = 1 [
    (google.api.field_behavior) = REQUIRED,
    (google.api.resource_reference) = {
      type: "aiplatform.googleapis.com/Index"
    }
  ];

  // A list of datapoint ids to be deleted.
  repeated string datapoint_ids = 2;
}

// Response message for
// [IndexService.RemoveDatapoints][google.cloud.aiplatform.v1beta1.IndexService.RemoveDatapoints]
message RemoveDatapointsResponse {}

// Runtime operation metadata with regard to Matching Engine Index.
message NearestNeighborSearchOperationMetadata {
  message RecordError {
    enum RecordErrorType {
      // Default, shall not be used.
      ERROR_TYPE_UNSPECIFIED = 0;

      // The record is empty.
      EMPTY_LINE = 1;

      // Invalid json format.
      INVALID_JSON_SYNTAX = 2;

      // Invalid csv format.
      INVALID_CSV_SYNTAX = 3;

      // Invalid avro format.
      INVALID_AVRO_SYNTAX = 4;

      // The embedding id is not valid.
      INVALID_EMBEDDING_ID = 5;

      // The size of the embedding vectors does not match with the specified
      // dimension.
      EMBEDDING_SIZE_MISMATCH = 6;

      // The `namespace` field is missing.
      NAMESPACE_MISSING = 7;

      // Generic catch-all error. Only used for validation failure where the
      // root cause cannot be easily retrieved programmatically.
      PARSING_ERROR = 8;

      // There are multiple restricts with the same `namespace` value.
      DUPLICATE_NAMESPACE = 9;

      // Numeric restrict has operator specified in datapoint.
      OP_IN_DATAPOINT = 10;

      // Numeric restrict has multiple values specified.
      MULTIPLE_VALUES = 11;

      // Numeric restrict has invalid numeric value specified.
      INVALID_NUMERIC_VALUE = 12;

      // File is not in UTF_8 format.
      INVALID_ENCODING = 13;
    }

    // The error type of this record.
    RecordErrorType error_type = 1;

    // A human-readable message that is shown to the user to help them fix the
    // error. Note that this message may change from time to time, your code
    // should check against error_type as the source of truth.
    string error_message = 2;

    // Cloud Storage URI pointing to the original file in user's bucket.
    string source_gcs_uri = 3;

    // Empty if the embedding id is failed to parse.
    string embedding_id = 4;

    // The original content of this record.
    string raw_record = 5;
  }

  message ContentValidationStats {
    // Cloud Storage URI pointing to the original file in user's bucket.
    string source_gcs_uri = 1;

    // Number of records in this file that were successfully processed.
    int64 valid_record_count = 2;

    // Number of records in this file we skipped due to validate errors.
    int64 invalid_record_count = 3;

    // The detail information of the partial failures encountered for those
    // invalid records that couldn't be parsed.
    // Up to 50 partial errors will be reported.
    repeated RecordError partial_errors = 4;
  }

  // The validation stats of the content (per file) to be inserted or
  // updated on the Matching Engine Index resource. Populated if
  // contentsDeltaUri is provided as part of
  // [Index.metadata][google.cloud.aiplatform.v1beta1.Index.metadata]. Please
  // note that, currently for those files that are broken or has unsupported
  // file format, we will not have the stats for those files.
  repeated ContentValidationStats content_validation_stats = 1;

  // The ingested data size in bytes.
  int64 data_bytes_count = 2;
}