aiplatform/v1/content.proto

// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package google.cloud.aiplatform.v1;

import "google/api/field_behavior.proto";
import "google/cloud/aiplatform/v1/tool.proto";
import "google/protobuf/duration.proto";
import "google/type/date.proto";

option csharp_namespace = "Google.Cloud.AIPlatform.V1";
option go_package = "cloud.google.com/go/aiplatform/apiv1/aiplatformpb;aiplatformpb";
option java_multiple_files = true;
option java_outer_classname = "ContentProto";
option java_package = "com.google.cloud.aiplatform.v1";
option php_namespace = "Google\\Cloud\\AIPlatform\\V1";
option ruby_package = "Google::Cloud::AIPlatform::V1";

// Harm categories that will block the content.
enum HarmCategory {
  // The harm category is unspecified.
  HARM_CATEGORY_UNSPECIFIED = 0;

  // The harm category is hate speech.
  HARM_CATEGORY_HATE_SPEECH = 1;

  // The harm category is dangerous content.
  HARM_CATEGORY_DANGEROUS_CONTENT = 2;

  // The harm category is harassment.
  HARM_CATEGORY_HARASSMENT = 3;

  // The harm category is sexually explicit content.
  HARM_CATEGORY_SEXUALLY_EXPLICIT = 4;
}

// The base structured datatype containing multi-part content of a message.
//
// A `Content` includes a `role` field designating the producer of the `Content`
// and a `parts` field containing multi-part data that contains the content of
// the message turn.
message Content {
  // Optional. The producer of the content. Must be either 'user' or 'model'.
  //
  // Useful to set for multi-turn conversations, otherwise can be left blank
  // or unset.
  string role = 1 [(google.api.field_behavior) = OPTIONAL];

  // Required. Ordered `Parts` that constitute a single message. Parts may have
  // different IANA MIME types.
  repeated Part parts = 2 [(google.api.field_behavior) = REQUIRED];
}

// A datatype containing media that is part of a multi-part `Content` message.
//
// A `Part` consists of data which has an associated datatype. A `Part` can only
// contain one of the accepted types in `Part.data`.
//
// A `Part` must have a fixed IANA MIME type identifying the type and subtype
// of the media if `inline_data` or `file_data` field is filled with raw bytes.
message Part {
  oneof data {
    // Optional. Text part (can be code).
    string text = 1 [(google.api.field_behavior) = OPTIONAL];

    // Optional. Inlined bytes data.
    Blob inline_data = 2 [(google.api.field_behavior) = OPTIONAL];

    // Optional. URI based data.
    FileData file_data = 3 [(google.api.field_behavior) = OPTIONAL];

    // Optional. A predicted [FunctionCall] returned from the model that
    // contains a string representing the [FunctionDeclaration.name] with the
    // parameters and their values.
    FunctionCall function_call = 5 [(google.api.field_behavior) = OPTIONAL];

    // Optional. The result output of a [FunctionCall] that contains a string
    // representing the [FunctionDeclaration.name] and a structured JSON object
    // containing any output from the function call. It is used as context to
    // the model.
    FunctionResponse function_response = 6
        [(google.api.field_behavior) = OPTIONAL];
  }

  oneof metadata {
    // Optional. Video metadata. The metadata should only be specified while the
    // video data is presented in inline_data or file_data.
    VideoMetadata video_metadata = 4 [(google.api.field_behavior) = OPTIONAL];
  }
}

// Content blob.
//
// It's preferred to send as [text][google.cloud.aiplatform.v1.Part.text]
// directly rather than raw bytes.
message Blob {
  // Required. The IANA standard MIME type of the source data.
  string mime_type = 1 [(google.api.field_behavior) = REQUIRED];

  // Required. Raw bytes.
  bytes data = 2 [(google.api.field_behavior) = REQUIRED];
}

// URI based data.
message FileData {
  // Required. The IANA standard MIME type of the source data.
  string mime_type = 1 [(google.api.field_behavior) = REQUIRED];

  // Required. URI.
  string file_uri = 2 [(google.api.field_behavior) = REQUIRED];
}

// Metadata describes the input video content.
message VideoMetadata {
  // Optional. The start offset of the video.
  google.protobuf.Duration start_offset = 1
      [(google.api.field_behavior) = OPTIONAL];

  // Optional. The end offset of the video.
  google.protobuf.Duration end_offset = 2
      [(google.api.field_behavior) = OPTIONAL];
}

// Generation config.
message GenerationConfig {
  // Optional. Controls the randomness of predictions.
  optional float temperature = 1 [(google.api.field_behavior) = OPTIONAL];

  // Optional. If specified, nucleus sampling will be used.
  optional float top_p = 2 [(google.api.field_behavior) = OPTIONAL];

  // Optional. If specified, top-k sampling will be used.
  optional float top_k = 3 [(google.api.field_behavior) = OPTIONAL];

  // Optional. Number of candidates to generate.
  optional int32 candidate_count = 4 [(google.api.field_behavior) = OPTIONAL];

  // Optional. The maximum number of output tokens to generate per message.
  optional int32 max_output_tokens = 5 [(google.api.field_behavior) = OPTIONAL];

  // Optional. Stop sequences.
  repeated string stop_sequences = 6 [(google.api.field_behavior) = OPTIONAL];

  // Optional. Positive penalties.
  optional float presence_penalty = 8 [(google.api.field_behavior) = OPTIONAL];

  // Optional. Frequency penalties.
  optional float frequency_penalty = 9 [(google.api.field_behavior) = OPTIONAL];

  // Optional. Output response mimetype of the generated candidate text.
  // Supported mimetype:
  // - `text/plain`: (default) Text output.
  // - `application/json`: JSON response in the candidates.
  // The model needs to be prompted to output the appropriate response type,
  // otherwise the behavior is undefined.
  // This is a preview feature.
  string response_mime_type = 13 [(google.api.field_behavior) = OPTIONAL];
}

// Safety settings.
message SafetySetting {
  // Probability based thresholds levels for blocking.
  enum HarmBlockThreshold {
    // Unspecified harm block threshold.
    HARM_BLOCK_THRESHOLD_UNSPECIFIED = 0;

    // Block low threshold and above (i.e. block more).
    BLOCK_LOW_AND_ABOVE = 1;

    // Block medium threshold and above.
    BLOCK_MEDIUM_AND_ABOVE = 2;

    // Block only high threshold (i.e. block less).
    BLOCK_ONLY_HIGH = 3;

    // Block none.
    BLOCK_NONE = 4;
  }

  // Probability vs severity.
  enum HarmBlockMethod {
    // The harm block method is unspecified.
    HARM_BLOCK_METHOD_UNSPECIFIED = 0;

    // The harm block method uses both probability and severity scores.
    SEVERITY = 1;

    // The harm block method uses the probability score.
    PROBABILITY = 2;
  }

  // Required. Harm category.
  HarmCategory category = 1 [(google.api.field_behavior) = REQUIRED];

  // Required. The harm block threshold.
  HarmBlockThreshold threshold = 2 [(google.api.field_behavior) = REQUIRED];

  // Optional. Specify if the threshold is used for probability or severity
  // score. If not specified, the threshold is used for probability score.
  HarmBlockMethod method = 4 [(google.api.field_behavior) = OPTIONAL];
}

// Safety rating corresponding to the generated content.
message SafetyRating {
  // Harm probability levels in the content.
  enum HarmProbability {
    // Harm probability unspecified.
    HARM_PROBABILITY_UNSPECIFIED = 0;

    // Negligible level of harm.
    NEGLIGIBLE = 1;

    // Low level of harm.
    LOW = 2;

    // Medium level of harm.
    MEDIUM = 3;

    // High level of harm.
    HIGH = 4;
  }

  // Harm severity levels.
  enum HarmSeverity {
    // Harm severity unspecified.
    HARM_SEVERITY_UNSPECIFIED = 0;

    // Negligible level of harm severity.
    HARM_SEVERITY_NEGLIGIBLE = 1;

    // Low level of harm severity.
    HARM_SEVERITY_LOW = 2;

    // Medium level of harm severity.
    HARM_SEVERITY_MEDIUM = 3;

    // High level of harm severity.
    HARM_SEVERITY_HIGH = 4;
  }

  // Output only. Harm category.
  HarmCategory category = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Harm probability levels in the content.
  HarmProbability probability = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Harm probability score.
  float probability_score = 5 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Harm severity levels in the content.
  HarmSeverity severity = 6 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Harm severity score.
  float severity_score = 7 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Indicates whether the content was filtered out because of this
  // rating.
  bool blocked = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// A collection of source attributions for a piece of content.
message CitationMetadata {
  // Output only. List of citations.
  repeated Citation citations = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// Source attributions for content.
message Citation {
  // Output only. Start index into the content.
  int32 start_index = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. End index into the content.
  int32 end_index = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Url reference of the attribution.
  string uri = 3 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Title of the attribution.
  string title = 4 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. License of the attribution.
  string license = 5 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Publication date of the attribution.
  google.type.Date publication_date = 6
      [(google.api.field_behavior) = OUTPUT_ONLY];
}

// A response candidate generated from the model.
message Candidate {
  // The reason why the model stopped generating tokens.
  // If empty, the model has not stopped generating the tokens.
  enum FinishReason {
    // The finish reason is unspecified.
    FINISH_REASON_UNSPECIFIED = 0;

    // Natural stop point of the model or provided stop sequence.
    STOP = 1;

    // The maximum number of tokens as specified in the request was reached.
    MAX_TOKENS = 2;

    // The token generation was stopped as the response was flagged for safety
    // reasons. NOTE: When streaming the Candidate.content will be empty if
    // content filters blocked the output.
    SAFETY = 3;

    // The token generation was stopped as the response was flagged for
    // unauthorized citations.
    RECITATION = 4;

    // All other reasons that stopped the token generation
    OTHER = 5;

    // The token generation was stopped as the response was flagged for the
    // terms which are included from the terminology blocklist.
    BLOCKLIST = 6;

    // The token generation was stopped as the response was flagged for
    // the prohibited contents.
    PROHIBITED_CONTENT = 7;

    // The token generation was stopped as the response was flagged for
    // Sensitive Personally Identifiable Information (SPII) contents.
    SPII = 8;
  }

  // Output only. Index of the candidate.
  int32 index = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Content parts of the candidate.
  Content content = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The reason why the model stopped generating tokens.
  // If empty, the model has not stopped generating the tokens.
  FinishReason finish_reason = 3 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. List of ratings for the safety of a response candidate.
  //
  // There is at most one rating per category.
  repeated SafetyRating safety_ratings = 4
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Describes the reason the mode stopped generating tokens in
  // more detail. This is only filled when `finish_reason` is set.
  optional string finish_message = 5
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Source attribution of the generated content.
  CitationMetadata citation_metadata = 6
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Metadata specifies sources used to ground generated content.
  GroundingMetadata grounding_metadata = 7
      [(google.api.field_behavior) = OUTPUT_ONLY];
}

// Segment of the content.
message Segment {
  // Output only. The index of a Part object within its parent Content object.
  int32 part_index = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Start index in the given Part, measured in bytes. Offset from
  // the start of the Part, inclusive, starting at zero.
  int32 start_index = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. End index in the given Part, measured in bytes. Offset from
  // the start of the Part, exclusive, starting at zero.
  int32 end_index = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// Grounding attribution.
message GroundingAttribution {
  // Attribution from the web.
  message Web {
    // Output only. URI reference of the attribution.
    string uri = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Title of the attribution.
    string title = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  oneof reference {
    // Optional. Attribution from the web.
    Web web = 3 [(google.api.field_behavior) = OPTIONAL];
  }

  // Output only. Segment of the content this attribution belongs to.
  Segment segment = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Optional. Output only. Confidence score of the attribution. Ranges from 0
  // to 1. 1 is the most confident.
  optional float confidence_score = 2 [
    (google.api.field_behavior) = OPTIONAL,
    (google.api.field_behavior) = OUTPUT_ONLY
  ];
}

// Metadata returned to client when grounding is enabled.
message GroundingMetadata {
  // Optional. Web search queries for the following-up web search.
  repeated string web_search_queries = 1
      [(google.api.field_behavior) = OPTIONAL];

  // Optional. List of grounding attributions.
  repeated GroundingAttribution grounding_attributions = 2
      [(google.api.field_behavior) = OPTIONAL];
}