1// Copyright 2023 Google LLC 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15syntax = "proto3"; 16 17package google.cloud.aiplatform.v1; 18 19import "google/api/field_behavior.proto"; 20import "google/api/resource.proto"; 21import "google/cloud/aiplatform/v1/explanation.proto"; 22import "google/protobuf/struct.proto"; 23import "google/protobuf/timestamp.proto"; 24 25option csharp_namespace = "Google.Cloud.AIPlatform.V1"; 26option go_package = "cloud.google.com/go/aiplatform/apiv1/aiplatformpb;aiplatformpb"; 27option java_multiple_files = true; 28option java_outer_classname = "ModelEvaluationProto"; 29option java_package = "com.google.cloud.aiplatform.v1"; 30option php_namespace = "Google\\Cloud\\AIPlatform\\V1"; 31option ruby_package = "Google::Cloud::AIPlatform::V1"; 32 33// A collection of metrics calculated by comparing Model's predictions on all of 34// the test data against annotations from the test data. 35message ModelEvaluation { 36 option (google.api.resource) = { 37 type: "aiplatform.googleapis.com/ModelEvaluation" 38 pattern: "projects/{project}/locations/{location}/models/{model}/evaluations/{evaluation}" 39 }; 40 41 message ModelEvaluationExplanationSpec { 42 // Explanation type. 43 // 44 // For AutoML Image Classification models, possible values are: 45 // 46 // * `image-integrated-gradients` 47 // * `image-xrai` 48 string explanation_type = 1; 49 50 // Explanation spec details. 51 ExplanationSpec explanation_spec = 2; 52 } 53 54 // Output only. The resource name of the ModelEvaluation. 55 string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; 56 57 // The display name of the ModelEvaluation. 58 string display_name = 10; 59 60 // Points to a YAML file stored on Google Cloud Storage describing the 61 // [metrics][google.cloud.aiplatform.v1.ModelEvaluation.metrics] of this 62 // ModelEvaluation. The schema is defined as an OpenAPI 3.0.2 [Schema 63 // Object](https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.2.md#schemaObject). 64 string metrics_schema_uri = 2; 65 66 // Evaluation metrics of the Model. The schema of the metrics is stored in 67 // [metrics_schema_uri][google.cloud.aiplatform.v1.ModelEvaluation.metrics_schema_uri] 68 google.protobuf.Value metrics = 3; 69 70 // Output only. Timestamp when this ModelEvaluation was created. 71 google.protobuf.Timestamp create_time = 4 72 [(google.api.field_behavior) = OUTPUT_ONLY]; 73 74 // All possible 75 // [dimensions][google.cloud.aiplatform.v1.ModelEvaluationSlice.Slice.dimension] 76 // of ModelEvaluationSlices. The dimensions can be used as the filter of the 77 // [ModelService.ListModelEvaluationSlices][google.cloud.aiplatform.v1.ModelService.ListModelEvaluationSlices] 78 // request, in the form of `slice.dimension = <dimension>`. 79 repeated string slice_dimensions = 5; 80 81 // Points to a YAML file stored on Google Cloud Storage describing 82 // [EvaluatedDataItemView.data_item_payload][] and 83 // [EvaluatedAnnotation.data_item_payload][google.cloud.aiplatform.v1.EvaluatedAnnotation.data_item_payload]. 84 // The schema is defined as an OpenAPI 3.0.2 [Schema 85 // Object](https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.2.md#schemaObject). 86 // 87 // This field is not populated if there are neither EvaluatedDataItemViews nor 88 // EvaluatedAnnotations under this ModelEvaluation. 89 string data_item_schema_uri = 6; 90 91 // Points to a YAML file stored on Google Cloud Storage describing 92 // [EvaluatedDataItemView.predictions][], 93 // [EvaluatedDataItemView.ground_truths][], 94 // [EvaluatedAnnotation.predictions][google.cloud.aiplatform.v1.EvaluatedAnnotation.predictions], 95 // and 96 // [EvaluatedAnnotation.ground_truths][google.cloud.aiplatform.v1.EvaluatedAnnotation.ground_truths]. 97 // The schema is defined as an OpenAPI 3.0.2 [Schema 98 // Object](https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.2.md#schemaObject). 99 // 100 // This field is not populated if there are neither EvaluatedDataItemViews nor 101 // EvaluatedAnnotations under this ModelEvaluation. 102 string annotation_schema_uri = 7; 103 104 // Aggregated explanation metrics for the Model's prediction output over the 105 // data this ModelEvaluation uses. This field is populated only if the Model 106 // is evaluated with explanations, and only for AutoML tabular Models. 107 // 108 ModelExplanation model_explanation = 8; 109 110 // Describes the values of 111 // [ExplanationSpec][google.cloud.aiplatform.v1.ExplanationSpec] that are used 112 // for explaining the predicted values on the evaluated data. 113 repeated ModelEvaluationExplanationSpec explanation_specs = 9; 114 115 // The metadata of the ModelEvaluation. 116 // For the ModelEvaluation uploaded from Managed Pipeline, metadata contains a 117 // structured value with keys of "pipeline_job_id", "evaluation_dataset_type", 118 // "evaluation_dataset_path", "row_based_metrics_path". 119 google.protobuf.Value metadata = 11; 120} 121