googleapis/google/ai/generativelanguage/v1/content.proto

// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package google.ai.generativelanguage.v1;

import "google/api/field_behavior.proto";
import "google/protobuf/duration.proto";

option go_package = "cloud.google.com/go/ai/generativelanguage/apiv1/generativelanguagepb;generativelanguagepb";
option java_multiple_files = true;
option java_outer_classname = "ContentProto";
option java_package = "com.google.ai.generativelanguage.v1";

// Content Part modality
enum Modality {
  // Unspecified modality.
  MODALITY_UNSPECIFIED = 0;

  // Plain text.
  TEXT = 1;

  // Image.
  IMAGE = 2;

  // Video.
  VIDEO = 3;

  // Audio.
  AUDIO = 4;

  // Document, e.g. PDF.
  DOCUMENT = 5;
}

// The base structured datatype containing multi-part content of a message.
//
// A `Content` includes a `role` field designating the producer of the `Content`
// and a `parts` field containing multi-part data that contains the content of
// the message turn.
message Content {
  // Ordered `Parts` that constitute a single message. Parts may have different
  // MIME types.
  repeated Part parts = 1;

  // Optional. The producer of the content. Must be either 'user' or 'model'.
  //
  // Useful to set for multi-turn conversations, otherwise can be left blank
  // or unset.
  string role = 2 [(google.api.field_behavior) = OPTIONAL];
}

// A datatype containing media that is part of a multi-part `Content` message.
//
// A `Part` consists of data which has an associated datatype. A `Part` can only
// contain one of the accepted types in `Part.data`.
//
// A `Part` must have a fixed IANA MIME type identifying the type and subtype
// of the media if the `inline_data` field is filled with raw bytes.
message Part {
  oneof data {
    // Inline text.
    string text = 2;

    // Inline media bytes.
    Blob inline_data = 3;
  }

  // Controls extra preprocessing of data.
  oneof metadata {
    // Optional. Video metadata. The metadata should only be specified while the
    // video data is presented in inline_data or file_data.
    VideoMetadata video_metadata = 14 [(google.api.field_behavior) = OPTIONAL];
  }
}

// Raw media bytes.
//
// Text should not be sent as raw bytes, use the 'text' field.
message Blob {
  // The IANA standard MIME type of the source data.
  // Examples:
  //   - image/png
  //   - image/jpeg
  // If an unsupported MIME type is provided, an error will be returned. For a
  // complete list of supported types, see [Supported file
  // formats](https://ai.google.dev/gemini-api/docs/prompting_with_media#supported_file_formats).
  string mime_type = 1;

  // Raw bytes for media formats.
  bytes data = 2;
}

// Metadata describes the input video content.
message VideoMetadata {
  // Optional. The start offset of the video.
  google.protobuf.Duration start_offset = 1
      [(google.api.field_behavior) = OPTIONAL];

  // Optional. The end offset of the video.
  google.protobuf.Duration end_offset = 2
      [(google.api.field_behavior) = OPTIONAL];

  // Optional. The frame rate of the video sent to the model. If not specified,
  // the default value will be 1.0. The fps range is (0.0, 24.0].
  double fps = 3 [(google.api.field_behavior) = OPTIONAL];
}

// Represents token counting info for a single modality.
message ModalityTokenCount {
  // The modality associated with this token count.
  Modality modality = 1;

  // Number of tokens.
  int32 token_count = 2;
}