aiplatform/v1/evaluated_annotation.proto

// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package google.cloud.aiplatform.v1;

import "google/api/field_behavior.proto";
import "google/cloud/aiplatform/v1/explanation.proto";
import "google/protobuf/struct.proto";

option csharp_namespace = "Google.Cloud.AIPlatform.V1";
option go_package = "cloud.google.com/go/aiplatform/apiv1/aiplatformpb;aiplatformpb";
option java_multiple_files = true;
option java_outer_classname = "EvaluatedAnnotationProto";
option java_package = "com.google.cloud.aiplatform.v1";
option php_namespace = "Google\\Cloud\\AIPlatform\\V1";
option ruby_package = "Google::Cloud::AIPlatform::V1";

// True positive, false positive, or false negative.
//
// EvaluatedAnnotation is only available under ModelEvaluationSlice with slice
// of `annotationSpec` dimension.
message EvaluatedAnnotation {
  // Describes the type of the EvaluatedAnnotation. The type is determined
  enum EvaluatedAnnotationType {
    // Invalid value.
    EVALUATED_ANNOTATION_TYPE_UNSPECIFIED = 0;

    // The EvaluatedAnnotation is a true positive. It has a prediction created
    // by the Model and a ground truth Annotation which the prediction matches.
    TRUE_POSITIVE = 1;

    // The EvaluatedAnnotation is false positive. It has a prediction created by
    // the Model which does not match any ground truth annotation.
    FALSE_POSITIVE = 2;

    // The EvaluatedAnnotation is false negative. It has a ground truth
    // annotation which is not matched by any of the model created predictions.
    FALSE_NEGATIVE = 3;
  }

  // Output only. Type of the EvaluatedAnnotation.
  EvaluatedAnnotationType type = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The model predicted annotations.
  //
  // For true positive, there is one and only one prediction, which matches the
  // only one ground truth annotation in
  // [ground_truths][google.cloud.aiplatform.v1.EvaluatedAnnotation.ground_truths].
  //
  // For false positive, there is one and only one prediction, which doesn't
  // match any ground truth annotation of the corresponding
  // [data_item_view_id][EvaluatedAnnotation.data_item_view_id].
  //
  // For false negative, there are zero or more predictions which are similar to
  // the only ground truth annotation in
  // [ground_truths][google.cloud.aiplatform.v1.EvaluatedAnnotation.ground_truths]
  // but not enough for a match.
  //
  // The schema of the prediction is stored in
  // [ModelEvaluation.annotation_schema_uri][google.cloud.aiplatform.v1.ModelEvaluation.annotation_schema_uri]
  repeated google.protobuf.Value predictions = 2
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The ground truth Annotations, i.e. the Annotations that exist
  // in the test data the Model is evaluated on.
  //
  // For true positive, there is one and only one ground truth annotation, which
  // matches the only prediction in
  // [predictions][google.cloud.aiplatform.v1.EvaluatedAnnotation.predictions].
  //
  // For false positive, there are zero or more ground truth annotations that
  // are similar to the only prediction in
  // [predictions][google.cloud.aiplatform.v1.EvaluatedAnnotation.predictions],
  // but not enough for a match.
  //
  // For false negative, there is one and only one ground truth annotation,
  // which doesn't match any predictions created by the model.
  //
  // The schema of the ground truth is stored in
  // [ModelEvaluation.annotation_schema_uri][google.cloud.aiplatform.v1.ModelEvaluation.annotation_schema_uri]
  repeated google.protobuf.Value ground_truths = 3
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The data item payload that the Model predicted this
  // EvaluatedAnnotation on.
  google.protobuf.Value data_item_payload = 5
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. ID of the EvaluatedDataItemView under the same ancestor
  // ModelEvaluation. The EvaluatedDataItemView consists of all ground truths
  // and predictions on
  // [data_item_payload][google.cloud.aiplatform.v1.EvaluatedAnnotation.data_item_payload].
  string evaluated_data_item_view_id = 6
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Explanations of
  // [predictions][google.cloud.aiplatform.v1.EvaluatedAnnotation.predictions].
  // Each element of the explanations indicates the explanation for one
  // explanation Method.
  //
  // The attributions list in the
  // [EvaluatedAnnotationExplanation.explanation][google.cloud.aiplatform.v1.EvaluatedAnnotationExplanation.explanation]
  // object corresponds to the
  // [predictions][google.cloud.aiplatform.v1.EvaluatedAnnotation.predictions]
  // list. For example, the second element in the attributions list explains the
  // second element in the predictions list.
  repeated EvaluatedAnnotationExplanation explanations = 8;

  // Annotations of model error analysis results.
  repeated ErrorAnalysisAnnotation error_analysis_annotations = 9;
}

// Explanation result of the prediction produced by the Model.
message EvaluatedAnnotationExplanation {
  // Explanation type.
  //
  // For AutoML Image Classification models, possible values are:
  //
  //   * `image-integrated-gradients`
  //   * `image-xrai`
  string explanation_type = 1;

  // Explanation attribution response details.
  Explanation explanation = 2;
}

// Model error analysis for each annotation.
message ErrorAnalysisAnnotation {
  // Attributed items for a given annotation, typically representing neighbors
  // from the training sets constrained by the query type.
  message AttributedItem {
    // The unique ID for each annotation. Used by FE to allocate the annotation
    // in DB.
    string annotation_resource_name = 1;

    // The distance of this item to the annotation.
    double distance = 2;
  }

  // The query type used for finding the attributed items.
  enum QueryType {
    // Unspecified query type for model error analysis.
    QUERY_TYPE_UNSPECIFIED = 0;

    // Query similar samples across all classes in the dataset.
    ALL_SIMILAR = 1;

    // Query similar samples from the same class of the input sample.
    SAME_CLASS_SIMILAR = 2;

    // Query dissimilar samples from the same class of the input sample.
    SAME_CLASS_DISSIMILAR = 3;
  }

  // Attributed items for a given annotation, typically representing neighbors
  // from the training sets constrained by the query type.
  repeated AttributedItem attributed_items = 1;

  // The query type used for finding the attributed items.
  QueryType query_type = 2;

  // The outlier score of this annotated item. Usually defined as the min of all
  // distances from attributed items.
  double outlier_score = 3;

  // The threshold used to determine if this annotation is an outlier or not.
  double outlier_threshold = 4;
}