speech.proto

// Copyright 2021 Tiro ehf.
// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package tiro.speech.v1alpha;

import "google/api/annotations.proto";
import "google/api/client.proto";
import "google/api/field_behavior.proto";
import "google/protobuf/duration.proto";
import "google/rpc/status.proto";

option cc_enable_arenas = true;
option go_package = "gitlab.com/tiro-is/tiro-apis-gogen/tiro/speech/v1alpha;speech";
option java_multiple_files = true;
option java_outer_classname = "SpeechProto";
option java_package = "is.tiro.speech.v1alpha";
option objc_class_prefix = "TS";

// Service that implements Google Cloud Speech API.
service Speech {
  option (google.api.default_host) = "speech.tiro.is";

  // Performs synchronous speech recognition: receive results after all audio
  // has been sent and processed.
  rpc Recognize(RecognizeRequest) returns (RecognizeResponse) {
    option (google.api.http) = {
      post: "/v1alpha/speech:recognize"
      body: "*"
    };
    option (google.api.method_signature) = "config,audio";
  }

  // Performs bidirectional streaming speech recognition: receive results while
  // sending audio. This method is only available via the gRPC API (not REST).
  rpc StreamingRecognize(stream StreamingRecognizeRequest)
      returns (stream StreamingRecognizeResponse) {}
}

// The top-level message sent by the client for the `Recognize` method.
message RecognizeRequest {
  // Required. Provides information to the recognizer that specifies how to
  // process the request.
  RecognitionConfig config = 1 [(google.api.field_behavior) = REQUIRED];

  // Required. The audio data to be recognized.
  RecognitionAudio audio = 2 [(google.api.field_behavior) = REQUIRED];
}

// The top-level message sent by the client for the `StreamingRecognize` method.
// Multiple `StreamingRecognizeRequest` messages are sent. The first message
// must contain a `streaming_config` message and must not contain
// `audio_content`. All subsequent messages must contain `audio_content` and
// must not contain a `streaming_config` message.
message StreamingRecognizeRequest {
  // The streaming request, which is either a streaming config or audio content.
  oneof streaming_request {
    // Provides information to the recognizer that specifies how to process the
    // request. The first `StreamingRecognizeRequest` message must contain a
    // `streaming_config`  message.
    StreamingRecognitionConfig streaming_config = 1;

    // The audio data to be recognized. Sequential chunks of audio data are sent
    // in sequential `StreamingRecognizeRequest` messages. The first
    // `StreamingRecognizeRequest` message must not contain `audio_content` data
    // and all subsequent `StreamingRecognizeRequest` messages must contain
    // `audio_content` data. The audio bytes must be encoded as specified in
    // `RecognitionConfig`. Note: as with all bytes fields, proto buffers use a
    // pure binary representation (not base64). See
    // [content limits](https://cloud.google.com/speech-to-text/quotas#content).
    bytes audio_content = 2;
  }
}

// Provides information to the recognizer that specifies how to process the
// request.
message StreamingRecognitionConfig {
  // Required. Provides information to the recognizer that specifies how to
  // process the request.
  RecognitionConfig config = 1 [(google.api.field_behavior) = REQUIRED];

  // If `false` or omitted, the recognizer will perform continuous
  // recognition (continuing to wait for and process audio even if the user
  // pauses speaking) until the client closes the input stream (gRPC API) or
  // until the maximum time limit has been reached. May return multiple
  // `StreamingRecognitionResult`s with the `is_final` flag set to `true`.
  //
  // If `true`, the recognizer will detect a single spoken utterance. When it
  // detects that the user has paused or stopped speaking, it will return an
  // `END_OF_SINGLE_UTTERANCE` event and cease recognition. It will return no
  // more than one `StreamingRecognitionResult` with the `is_final` flag set to
  // `true`.
  bool single_utterance = 2;

  // If `true`, interim results (tentative hypotheses) may be
  // returned as they become available (these interim results are indicated with
  // the `is_final=false` flag).
  // If `false` or omitted, only `is_final=true` result(s) are returned.
  bool interim_results = 3;
}

// Provides information to the recognizer that specifies how to process the
// request.
message RecognitionConfig {
  // The encoding of the audio data sent in the request.
  //
  // All encodings support only 1 channel (mono) audio
  //
  // For best results, the audio source should be captured and transmitted using
  // a lossless encoding (`FLAC` or `LINEAR16`). The accuracy of the speech
  // recognition can be reduced if lossy codecs are used to capture or transmit
  // audio.
  enum AudioEncoding {
    // Not specified.
    ENCODING_UNSPECIFIED = 0;

    // Uncompressed 16-bit signed little-endian samples (Linear PCM).
    LINEAR16 = 1;

    // `FLAC` (Free Lossless Audio
    // Codec) is the recommended encoding because it is
    // lossless--therefore recognition is not compromised--and
    // requires only about half the bandwidth of `LINEAR16`. `FLAC` stream
    // encoding supports 16-bit and 24-bit samples, however, not all fields in
    // `STREAMINFO` are supported.
    FLAC = 2;

    // MP3 audio. Support all standard MP3 bitrates (which range from 32-320
    // kbps). When using this encoding, `sample_rate_hertz` has to match the
    // sample rate of the file being used.
    MP3 = 8;
  }

  // Encoding of audio data sent in all `RecognitionAudio` messages.
  // This field is optional for `FLAC` and `WAV` audio files and required
  // for all other audio formats. For details, see
  // [AudioEncoding][tiro.speech.v1alpha.RecognitionConfig.AudioEncoding].
  AudioEncoding encoding = 1;

  // Sample rate in Hertz of the audio data sent in all
  // `RecognitionAudio` messages. Valid values are: 8000-48000.
  // 16000 is optimal. For best results, set the sampling rate of the audio
  // source to 16000 Hz. If that's not possible, use the native sample rate of
  // the audio source (instead of re-sampling).
  // [AudioEncoding][tiro.speech.v1alpha.RecognitionConfig.AudioEncoding].
  int32 sample_rate_hertz = 2;

  // Required. The language of the supplied audio as a
  // [BCP-47](https://www.rfc-editor.org/rfc/bcp/bcp47.txt) language tag.
  // Example: "is-IS".
  string language_code = 3 [(google.api.field_behavior) = REQUIRED];

  // Maximum number of recognition hypotheses to be returned.
  // Specifically, the maximum number of `SpeechRecognitionAlternative` messages
  // within each `SpeechRecognitionResult`.
  // The server may return fewer than `max_alternatives`.
  // Valid values are `0`-`30`. A value of `0` or `1` will return a maximum of
  // one. If omitted, will return a maximum of one.
  int32 max_alternatives = 4;

  // If `true`, the top result includes a list of words and
  // the start and end time offsets (timestamps) for those words. If
  // `false`, no word-level time offset information is returned. The default is
  // `false`.
  bool enable_word_time_offsets = 8;

  // Metadata regarding this request.
  RecognitionMetadata metadata = 9;

  // If 'true', adds punctuation to recognition result hypotheses.
  bool enable_automatic_punctuation = 11;

  reserved 7, 12;

  SpeakerDiarizationConfig diarization_config = 19;
}

// Config to enable speaker diarization.
message SpeakerDiarizationConfig {
  // If 'true', enables speaker detection for each recognized word in
  // the top alternative of the recognition result using a speaker_tag provided
  // in the WordInfo.
  bool enable_speaker_diarization = 1;

  // Minimum number of speakers in the conversation. This range gives you more
  // flexibility by allowing the system to automatically determine the correct
  // number of speakers. If not set, the default value is 2.
  int32 min_speaker_count = 2;

  // Maximum number of speakers in the conversation. This range gives you more
  // flexibility by allowing the system to automatically determine the correct
  // number of speakers. If not set, the default value is 6.
  int32 max_speaker_count = 3;

  reserved "speaker_tag";
  reserved 5;
}

// Description of audio data to be recognized.
message RecognitionMetadata {
  // Use case categories that the audio recognition request can be described
  // by.
  enum InteractionType {
    // Use case is either unknown or is something other than one of the other
    // values below.
    INTERACTION_TYPE_UNSPECIFIED = 0;

    // Multiple people in a conversation or discussion. For example in a
    // meeting with two or more people actively participating. Typically
    // all the primary people speaking would be in the same room (if not,
    // see PHONE_CALL)
    DISCUSSION = 1;

    // One or more persons lecturing or presenting to others, mostly
    // uninterrupted.
    PRESENTATION = 2;

    // A phone-call or video-conference in which two or more people, who are
    // not in the same room, are actively participating.
    PHONE_CALL = 3;

    // A recorded message intended for another person to listen to.
    VOICEMAIL = 4;

    // Professionally produced audio (eg. TV Show, Podcast).
    PROFESSIONALLY_PRODUCED = 5;

    // Transcribe spoken questions and queries into text.
    VOICE_SEARCH = 6;

    // Transcribe voice commands, such as for controlling a device.
    VOICE_COMMAND = 7;

    // Transcribe speech to text to create a written document, such as a
    // text-message, email or report.
    DICTATION = 8;
  }

  // Enumerates the types of capture settings describing an audio file.
  enum MicrophoneDistance {
    // Audio type is not known.
    MICROPHONE_DISTANCE_UNSPECIFIED = 0;

    // The audio was captured from a closely placed microphone. Eg. phone,
    // dictaphone, or handheld microphone. Generally if there speaker is within
    // 1 meter of the microphone.
    NEARFIELD = 1;

    // The speaker if within 3 meters of the microphone.
    MIDFIELD = 2;

    // The speaker is more than 3 meters away from the microphone.
    FARFIELD = 3;
  }

  // The original media the speech was recorded on.
  enum OriginalMediaType {
    // Unknown original media type.
    ORIGINAL_MEDIA_TYPE_UNSPECIFIED = 0;

    // The speech data is an audio recording.
    AUDIO = 1;

    // The speech data originally recorded on a video.
    VIDEO = 2;
  }

  // The type of device the speech was recorded with.
  enum RecordingDeviceType {
    // The recording device is unknown.
    RECORDING_DEVICE_TYPE_UNSPECIFIED = 0;

    // Speech was recorded on a smartphone.
    SMARTPHONE = 1;

    // Speech was recorded using a personal computer or tablet.
    PC = 2;

    // Speech was recorded over a phone line.
    PHONE_LINE = 3;

    // Speech was recorded in a vehicle.
    VEHICLE = 4;

    // Speech was recorded outdoors.
    OTHER_OUTDOOR_DEVICE = 5;

    // Speech was recorded indoors.
    OTHER_INDOOR_DEVICE = 6;
  }

  // The use case most closely describing the audio content to be recognized.
  InteractionType interaction_type = 1;

  // The industry vertical to which this speech recognition request most
  // closely applies. This is most indicative of the topics contained
  // in the audio.  Use the 6-digit NAICS code to identify the industry
  // vertical - see https://www.naics.com/search/.
  uint32 industry_naics_code_of_audio = 3;

  // The audio type that most closely describes the audio being recognized.
  MicrophoneDistance microphone_distance = 4;

  // The original media the speech was recorded on.
  OriginalMediaType original_media_type = 5;

  // The type of device the speech was recorded with.
  RecordingDeviceType recording_device_type = 6;

  // The device used to make the recording.  Examples 'Nexus 5X' or
  // 'Polycom SoundStation IP 6000' or 'POTS' or 'VoIP' or
  // 'Cardioid Microphone'.
  string recording_device_name = 7;

  // Mime type of the original audio file.  For example `audio/m4a`,
  // `audio/x-alaw-basic`, `audio/mp3`, `audio/3gpp`.
  // A list of possible audio mime types is maintained at
  // http://www.iana.org/assignments/media-types/media-types.xhtml#audio
  string original_mime_type = 8;

  // Description of the content. Eg. "Recordings of federal supreme court
  // hearings from 2012".
  string audio_topic = 10;
}

// Contains audio data in the encoding specified in the `RecognitionConfig`.
// Either `content` or `uri` must be supplied. Supplying both or neither
// returns [google.rpc.Code.INVALID_ARGUMENT][google.rpc.Code.INVALID_ARGUMENT].
// See [content limits](https://cloud.google.com/speech-to-text/quotas#content).
message RecognitionAudio {
  // The audio source, which is either inline content or a Google Cloud
  // Storage uri.
  oneof audio_source {
    // The audio data bytes encoded as specified in
    // `RecognitionConfig`. Note: as with all bytes fields, proto buffers use a
    // pure binary representation, whereas JSON representations use base64.
    bytes content = 1;

    // URI that points to a file that contains audio data bytes as specified in
    // `RecognitionConfig`. The file must not be compressed (for example, gzip).
    string uri = 2;
  }
}

// The only message returned to the client by the `Recognize` method. It
// contains the result as zero or more sequential `SpeechRecognitionResult`
// messages.
message RecognizeResponse {
  // Sequential list of transcription results corresponding to
  // sequential portions of audio.
  repeated SpeechRecognitionResult results = 2;
}

// `StreamingRecognizeResponse` is the only message returned to the client by
// `StreamingRecognize`. A series of zero or more `StreamingRecognizeResponse`
// messages are streamed back to the client. If there is no recognizable
// audio, and `single_utterance` is set to false, then no messages are streamed
// back to the client.
//
// Here's an example of a series of ten `StreamingRecognizeResponse`s that might
// be returned while processing audio:
//
// 1. results { alternatives { transcript: "tube" } stability: 0.01 }
//
// 2. results { alternatives { transcript: "to be a" } stability: 0.01 }
//
// 3. results { alternatives { transcript: "to be" } stability: 0.9 }
//    results { alternatives { transcript: " or not to be" } stability: 0.01 }
//
// 4. results { alternatives { transcript: "to be or not to be"
//                             confidence: 0.92 }
//              alternatives { transcript: "to bee or not to bee" }
//              is_final: true }
//
// 5. results { alternatives { transcript: " that's" } stability: 0.01 }
//
// 6. results { alternatives { transcript: " that is" } stability: 0.9 }
//    results { alternatives { transcript: " the question" } stability: 0.01 }
//
// 7. results { alternatives { transcript: " that is the question"
//                             confidence: 0.98 }
//              alternatives { transcript: " that was the question" }
//              is_final: true }
//
// Notes:
//
// - Only two of the above responses #4 and #7 contain final results; they are
//   indicated by `is_final: true`. Concatenating these together generates the
//   full transcript: "to be or not to be that is the question".
//
// - The others contain interim `results`. #3 and #6 contain two interim
//   `results`: the first portion has a high stability and is less likely to
//   change; the second portion has a low stability and is very likely to
//   change. A UI designer might choose to show only high stability `results`.
//
// - The specific `stability` and `confidence` values shown above are only for
//   illustrative purposes. Actual values may vary.
//
// - In each response, only one of these fields will be set:
//     `error`,
//     `speech_event_type`, or
//     one or more (repeated) `results`.
message StreamingRecognizeResponse {
  // Indicates the type of speech event.
  enum SpeechEventType {
    // No speech event specified.
    SPEECH_EVENT_UNSPECIFIED = 0;

    // This event indicates that the server has detected the end of the user's
    // speech utterance and expects no additional speech. Therefore, the server
    // will not process additional audio (although it may subsequently return
    // additional results). The client should stop sending additional audio
    // data, half-close the gRPC connection, and wait for any additional results
    // until the server closes the gRPC connection. This event is only sent if
    // `single_utterance` was set to `true`, and is not used otherwise.
    END_OF_SINGLE_UTTERANCE = 1;
  }

  // If set, returns a [google.rpc.Status][google.rpc.Status] message that
  // specifies the error for the operation.
  google.rpc.Status error = 1;

  // This repeated list contains zero or more results that
  // correspond to consecutive portions of the audio currently being processed.
  // It contains zero or one `is_final=true` result (the newly settled portion),
  // followed by zero or more `is_final=false` results (the interim results).
  repeated StreamingRecognitionResult results = 2;

  // Indicates the type of speech event.
  SpeechEventType speech_event_type = 4;
}

// A streaming speech recognition result corresponding to a portion of the audio
// that is currently being processed.
message StreamingRecognitionResult {
  // May contain one or more recognition hypotheses (up to the
  // maximum specified in `max_alternatives`).
  // These alternatives are ordered in terms of accuracy, with the top (first)
  // alternative being the most probable, as ranked by the recognizer.
  repeated SpeechRecognitionAlternative alternatives = 1;

  // If `false`, this `StreamingRecognitionResult` represents an
  // interim result that may change. If `true`, this is the final time the
  // speech service will return this particular `StreamingRecognitionResult`,
  // the recognizer will not return any further hypotheses for this portion of
  // the transcript and corresponding audio.
  bool is_final = 2;

  // An estimate of the likelihood that the recognizer will not
  // change its guess about this interim result. Values range from 0.0
  // (completely unstable) to 1.0 (completely stable).
  // This field is only provided for interim results (`is_final=false`).
  // The default of 0.0 is a sentinel value indicating `stability` was not set.
  float stability = 3;

  reserved "result_end_time", "channel_tag", "language_code";
  reserved 4, 5, 6;
}

// A speech recognition result corresponding to a portion of the audio.
message SpeechRecognitionResult {
  // May contain one or more recognition hypotheses (up to the
  // maximum specified in `max_alternatives`).
  // These alternatives are ordered in terms of accuracy, with the top (first)
  // alternative being the most probable, as ranked by the recognizer.
  repeated SpeechRecognitionAlternative alternatives = 1;

  reserved 2, 5;
}

// Alternative hypotheses (a.k.a. n-best list).
message SpeechRecognitionAlternative {
  // Transcript text representing the words that the user spoke.
  string transcript = 1;

  // The confidence estimate between 0.0 and 1.0. A higher number
  // indicates an estimated greater likelihood that the recognized words are
  // correct. This field is set only for the top alternative of a non-streaming
  // result or, of a streaming result where `is_final=true`.
  // This field is not guaranteed to be accurate and users should not rely on it
  // to be always provided.
  // The default of 0.0 is a sentinel value indicating `confidence` was not set.
  float confidence = 2;

  // A list of word-specific information for each recognized word.
  // Note: When `enable_speaker_diarization` is true, you will see all the words
  // from the beginning of the audio.
  repeated WordInfo words = 3;
}

// Word-specific information for recognized words.
message WordInfo {
  // Time offset relative to the beginning of the audio,
  // and corresponding to the start of the spoken word.
  // This field is only set if `enable_word_time_offsets=true` and only
  // in the top hypothesis.
  google.protobuf.Duration start_time = 1;

  // Time offset relative to the beginning of the audio,
  // and corresponding to the end of the spoken word.
  // This field is only set if `enable_word_time_offsets=true` and only
  // in the top hypothesis.
  google.protobuf.Duration end_time = 2;

  // The word corresponding to this set of information.
  string word = 3;

  // The confidence estimate between 0.0 and 1.0. A higher number
  // indicates an estimated greater likelihood that the recognized words are
  // correct. This field is set only for the top alternative of a non-streaming
  // result or, of a streaming result where `is_final=true`.
  // This field is not guaranteed to be accurate and users should not rely on it
  // to be always provided.
  // The default of 0.0 is a sentinel value indicating `confidence` was not set.
  float confidence = 4;

  // A distinct integer value is assigned for every speaker within
  // the audio. This field specifies which one of those speakers was detected to
  // have spoken this word. Value ranges from '1' to diarization_speaker_count.
  // speaker_tag is set if enable_speaker_diarization = 'true' and only in the
  // top alternative.
  int32 speaker_tag = 5
    [(google.api.field_behavior) = OUTPUT_ONLY];
}