Unlocking Multimodal Video Transcription with Gemini

Requirement:

• A Google Cloud undertaking
• The Vertex AI API have to be enabled for this undertaking

Gen AI SDK atmosphere variables:

Be taught extra about establishing a undertaking and a improvement atmosphere.

Requirement:

Gen AI SDK atmosphere variables:

• GOOGLE_GENAI_USE_VERTEXAI="False"
• GOOGLE_API_KEY=""

Be taught extra about getting a Gemini API key from Google AI Studio.

import os
import sys
from collections.abc import Callable

from google import genai

# Guide setup (go away unchanged if setup is environment-defined)

# @markdown **Which API: Vertex AI or Google AI Studio?**
GOOGLE_GENAI_USE_VERTEXAI = True  # @param {sort: "boolean"}

# @markdown **Possibility A - Google Cloud undertaking [+location]**
GOOGLE_CLOUD_PROJECT = ""  # @param {sort: "string"}
GOOGLE_CLOUD_LOCATION = "international"  # @param {sort: "string"}

# @markdown **Possibility B - Google AI Studio API key**
GOOGLE_API_KEY = ""  # @param {sort: "string"}


def check_environment() -> bool:
    check_colab_user_authentication()
    return check_manual_setup() or check_vertex_ai() or check_colab() or check_local()


def check_manual_setup() -> bool:
    return check_define_env_vars(
        GOOGLE_GENAI_USE_VERTEXAI,
        GOOGLE_CLOUD_PROJECT.strip(),  # May need been pasted with line return
        GOOGLE_CLOUD_LOCATION,
        GOOGLE_API_KEY,
    )


def check_vertex_ai() -> bool:
    # Workbench and Colab Enterprise
    match os.getenv("VERTEX_PRODUCT", ""):
        case "WORKBENCH_INSTANCE":
            move
        case "COLAB_ENTERPRISE":
            if not running_in_colab_env():
                return False
        case _:
            return False

    return check_define_env_vars(
        True,
        os.getenv("GOOGLE_CLOUD_PROJECT", ""),
        os.getenv("GOOGLE_CLOUD_REGION", ""),
        "",
    )


def check_colab() -> bool:
    if not running_in_colab_env():
        return False

    # Colab Enterprise was checked earlier than, so that is Colab solely
    from google.colab import auth as colab_auth  # sort: ignore

    colab_auth.authenticate_user()

    # Use Colab Secrets and techniques (🗝️ icon in left panel) to retailer the atmosphere variables
    # Secrets and techniques are personal, seen solely to you and the notebooks that you choose
    # - Vertex AI: Retailer your settings as secrets and techniques
    # - Google AI: Straight import your Gemini API key from the UI
    vertexai, undertaking, location, api_key = get_vars(get_colab_secret)

    return check_define_env_vars(vertexai, undertaking, location, api_key)


def check_local() -> bool:
    vertexai, undertaking, location, api_key = get_vars(os.getenv)

    return check_define_env_vars(vertexai, undertaking, location, api_key)


def running_in_colab_env() -> bool:
    # Colab or Colab Enterprise
    return "google.colab" in sys.modules


def check_colab_user_authentication() -> None:
    if running_in_colab_env():
        from google.colab import auth as colab_auth  # sort: ignore

        colab_auth.authenticate_user()


def get_colab_secret(secret_name: str, default: str) -> str:
    from google.colab import userdata  # sort: ignore

    strive:
        return userdata.get(secret_name)
    besides Exception as e:
        return default


def get_vars(getenv: Callable[[str, str], str]) -> tuple[bool, str, str, str]:
    # Restrict getenv calls to the minimal (might set off UI affirmation for secret entry)
    vertexai_str = getenv("GOOGLE_GENAI_USE_VERTEXAI", "")
    if vertexai_str:
        vertexai = vertexai_str.decrease() in ["true", "1"]
    else:
        vertexai = bool(getenv("GOOGLE_CLOUD_PROJECT", ""))

    undertaking = getenv("GOOGLE_CLOUD_PROJECT", "") if vertexai else ""
    location = getenv("GOOGLE_CLOUD_LOCATION", "") if undertaking else ""
    api_key = getenv("GOOGLE_API_KEY", "") if not undertaking else ""

    return vertexai, undertaking, location, api_key


def check_define_env_vars(
    vertexai: bool,
    undertaking: str,
    location: str,
    api_key: str,
) -> bool:
    match (vertexai, bool(undertaking), bool(location), bool(api_key)):
        case (True, True, _, _):
            # Vertex AI - Google Cloud undertaking [+location]
            location = location or "international"
            define_env_vars(vertexai, undertaking, location, "")
        case (True, False, _, True):
            # Vertex AI - API key
            define_env_vars(vertexai, "", "", api_key)
        case (False, _, _, True):
            # Google AI Studio - API key
            define_env_vars(vertexai, "", "", api_key)
        case _:
            return False

    return True


def define_env_vars(vertexai: bool, undertaking: str, location: str, api_key: str) -> None:
    os.environ["GOOGLE_GENAI_USE_VERTEXAI"] = str(vertexai)
    os.environ["GOOGLE_CLOUD_PROJECT"] = undertaking
    os.environ["GOOGLE_CLOUD_LOCATION"] = location
    os.environ["GOOGLE_API_KEY"] = api_key


def check_configuration(consumer: genai.Shopper) -> None:
    service = "Vertex AI" if consumer.vertexai else "Google AI Studio"
    print(f"Utilizing the {service} API", finish="")

    if consumer._api_client.undertaking:
        print(f' with undertaking "{consumer._api_client.undertaking[:7]}…"', finish="")
        print(f' in location "{consumer._api_client.location}"')
    elif consumer._api_client.api_key:
        api_key = consumer._api_client.api_key
        print(f' with API key "{api_key[:5]}…{api_key[-5:]}"', finish="")
        print(f" (in case of error, be sure it was created for {service})")

import enum
from dataclasses import dataclass
from datetime import timedelta

import IPython.show
import tenacity
from google.genai.errors import ClientError
from google.genai.sorts import (
    FileData,
    FinishReason,
    GenerateContentConfig,
    GenerateContentResponse,
    Half,
    VideoMetadata,
)


class Mannequin(enum.Enum):
    # Typically Obtainable (GA)
    GEMINI_2_0_FLASH = "gemini-2.0-flash"
    GEMINI_2_5_FLASH = "gemini-2.5-flash"
    GEMINI_2_5_PRO = "gemini-2.5-pro"
    # Default mannequin
    DEFAULT = GEMINI_2_0_FLASH


# Default configuration for extra deterministic outputs
DEFAULT_CONFIG = GenerateContentConfig(
    temperature=0.0,
    top_p=0.0,
    seed=42,  # Arbitrary mounted worth
)

YOUTUBE_URL_PREFIX = "https://www.youtube.com/watch?v="
CLOUD_STORAGE_URI_PREFIX = "gs://"


def url_for_youtube_id(youtube_id: str) -> str:
    return f"{YOUTUBE_URL_PREFIX}{youtube_id}"


class Video(enum.Enum):
    move


class TestVideo(Video):
    # For testing functions, video period is statically specified within the enum identify
    # Suffix (ISO 8601 based mostly): _PT[H][M][S]

    # Google DeepMind | The Podcast | Season 3 Trailer | 59s
    GDM_PODCAST_TRAILER_PT59S = url_for_youtube_id("0pJn3g8dfwk")
    # Google Maps | Stroll within the footsteps of Jane Goodall | 2min 42s
    JANE_GOODALL_PT2M42S = "gs://cloud-samples-data/video/JaneGoodall.mp4"
    # Google DeepMind | AlphaFold | The making of a scientific breakthrough | 7min 54s
    GDM_ALPHAFOLD_PT7M54S = url_for_youtube_id("gg7WjuFs8F4")
    # Brut | French reportage | 8min 28s
    BRUT_FR_DOGS_WATER_LEAK_PT8M28S = url_for_youtube_id("U_yYkb-ureI")
    # Google DeepMind | The Podcast | AI for science | 54min 23s
    GDM_AI_FOR_SCIENCE_FRONTIER_PT54M23S = url_for_youtube_id("nQKmVhLIGcs")
    # Google I/O 2025 | Developer Keynote | 1h 10min 03s
    GOOGLE_IO_DEV_KEYNOTE_PT1H10M03S = url_for_youtube_id("GjvgtwSOCao")
    # Google Cloud | Subsequent 2025 | Opening Keynote | 1h 40min 03s
    GOOGLE_CLOUD_NEXT_PT1H40M03S = url_for_youtube_id("Md4Fs-Zc3tg")
    # Google I/O 2025 | Keynote | 1h 56min 35s
    GOOGLE_IO_KEYNOTE_PT1H56M35S = url_for_youtube_id("o8NiE3XMPrM")


class ShowAs(enum.Enum):
    DONT_SHOW = enum.auto()
    TEXT = enum.auto()
    MARKDOWN = enum.auto()


@dataclass
class VideoSegment:
    begin: timedelta
    finish: timedelta


def generate_content(
    immediate: str,
    video: Video | None = None,
    video_segment: VideoSegment | None = None,
    mannequin: Mannequin | None = None,
    config: GenerateContentConfig | None = None,
    show_as: ShowAs = ShowAs.TEXT,
) -> None:
    immediate = immediate.strip()
    mannequin = mannequin or Mannequin.DEFAULT
    config = config or DEFAULT_CONFIG

    model_id = mannequin.worth
    if video:
        if not (video_part := get_video_part(video, video_segment)):
            return
        contents = [video_part, prompt]
        caption = f"{video.identify} / {model_id}"
    else:
        contents = immediate
        caption = f"{model_id}"
    print(f" {caption} ".heart(80, "-"))

    for try in get_retrier():
        with try:
            response = consumer.fashions.generate_content(
                mannequin=model_id,
                contents=contents,
                config=config,
            )
            display_response_info(response)
            display_response(response, show_as)


def get_video_part(
    video: Video,
    video_segment: VideoSegment | None = None,
    fps: float | None = None,
) -> Half | None:
    video_uri: str = video.worth

    if not consumer.vertexai:
        video_uri = convert_to_https_url_if_cloud_storage_uri(video_uri)
        if not video_uri.startswith(YOUTUBE_URL_PREFIX):
            print("Google AI Studio API: Solely YouTube URLs are at the moment supported")
            return None

    file_data = FileData(file_uri=video_uri, mime_type="video/*")
    video_metadata = get_video_part_metadata(video_segment, fps)

    return Half(file_data=file_data, video_metadata=video_metadata)


def get_video_part_metadata(
    video_segment: VideoSegment | None = None,
    fps: float | None = None,
) -> VideoMetadata:
    def offset_as_str(offset: timedelta) -> str:
        return f"{offset.total_seconds()}s"

    if video_segment:
        start_offset = offset_as_str(video_segment.begin)
        end_offset = offset_as_str(video_segment.finish)
    else:
        start_offset = None
        end_offset = None

    return VideoMetadata(start_offset=start_offset, end_offset=end_offset, fps=fps)


def convert_to_https_url_if_cloud_storage_uri(uri: str) -> str:
    if uri.startswith(CLOUD_STORAGE_URI_PREFIX):
        return f"https://storage.googleapis.com/{uri.removeprefix(CLOUD_STORAGE_URI_PREFIX)}"
    else:
        return uri


def get_retrier() -> tenacity.Retrying:
    return tenacity.Retrying(
        cease=tenacity.stop_after_attempt(7),
        wait=tenacity.wait_incrementing(begin=10, increment=1),
        retry=should_retry_request,
        reraise=True,
    )


def should_retry_request(retry_state: tenacity.RetryCallState) -> bool:
    if not retry_state.final result:
        return False
    err = retry_state.final result.exception()
    if not isinstance(err, ClientError):
        return False
    print(f"❌ ClientError {err.code}: {err.message}")

    retry = False
    match err.code:
        case 400 if err.message is just not None and " strive once more " in err.message:
            # Workshop: undertaking accessing Cloud Storage for the primary time (service agent provisioning)
            retry = True
        case 429:
            # Workshop: non permanent undertaking with 1 QPM quota
            retry = True
    print(f"🔄 Retry: {retry}")

    return retry


def display_response_info(response: GenerateContentResponse) -> None:
    if usage_metadata := response.usage_metadata:
        if usage_metadata.prompt_token_count:
            print(f"Enter tokens   : {usage_metadata.prompt_token_count:9,d}")
        if usage_metadata.candidates_token_count:
            print(f"Output tokens  : {usage_metadata.candidates_token_count:9,d}")
        if usage_metadata.thoughts_token_count:
            print(f"Ideas tokens: {usage_metadata.thoughts_token_count:9,d}")
    if not response.candidates:
        print("❌ No `response.candidates`")
        return
    if (finish_reason := response.candidates[0].finish_reason) != FinishReason.STOP:
        print(f"❌ {finish_reason = }")
    if not response.textual content:
        print("❌ No `response.textual content`")
        return


def display_response(
    response: GenerateContentResponse,
    show_as: ShowAs,
) -> None:
    if show_as == ShowAs.DONT_SHOW:
        return
    if not (response_text := response.textual content):
        return
    response_text = response.textual content.strip()

    print(" begin of response ".heart(80, "-"))
    match show_as:
        case ShowAs.TEXT:
            print(response_text)
        case ShowAs.MARKDOWN:
            display_markdown(response_text)
    print(" finish of response ".heart(80, "-"))


def display_markdown(markdown: str) -> None:
    IPython.show.show(IPython.show.Markdown(markdown))


def display_video(video: Video) -> None:
    video_url = convert_to_https_url_if_cloud_storage_uri(video.worth)
    assert video_url.startswith("https://")

    video_width = 600
    if video_url.startswith(YOUTUBE_URL_PREFIX):
        youtube_id = video_url.removeprefix(YOUTUBE_URL_PREFIX)
        ipython_video = IPython.show.YouTubeVideo(youtube_id, width=video_width)
    else:
        ipython_video = IPython.show.Video(video_url, width=video_width)

    display_markdown(f"### Video ([source]({video_url}))")
    IPython.show.show(ipython_video)

------------------- GDM_ALPHAFOLD_PT7M54S / gemini-2.0-flash -------------------
Enter tokens   :   134,177
Output tokens  :     2,689
------------------------------ begin of response -------------------------------
[00:02] ? - We have found extra concerning the world than some other civilization earlier than us.
[00:08] ? - However we have now been caught on this one drawback.
[00:11] ? - How do proteins fold up?
[00:13] ? - How do proteins go from a string of amino acids to a compact form that acts as a machine and drives life?
[00:22] ? - If you discover out about proteins, it is rather thrilling.
[00:25] ? - You possibly can consider them as little organic nano machines.
[00:28] ? - They're basically the elemental constructing blocks that energy every little thing dwelling on this planet.
[00:34] ? - If we are able to reliably predict protein constructions utilizing AI, that might change the way in which we perceive the pure world.
[00:46] ? - Protein folding is certainly one of these holy grail sort issues in biology.
[00:50] Demis Hassabis - We have at all times hypothesized that AI must be useful to make these varieties of huge scientific breakthroughs extra rapidly.
[00:58] ? - After which I am going to in all probability be taking a look at little tunings which may make a distinction.
[01:02] ? - It must be making a histogram on and a background talent.
[01:04] ? - We have been engaged on our system AlphaFold actually arduous now for over two years.
[01:08] ? - Slightly than having to do painstaking experiments, sooner or later biologists may have the ability to as an alternative depend on AI strategies to instantly predict constructions rapidly and effectively.
[01:17] Kathryn Tunyasuvunakool - Typically talking, biologists are usually fairly skeptical of computational work, and I feel that skepticism is wholesome and I respect it, however I really feel very enthusiastic about what AlphaFold can obtain.
[01:28] Andrew Senior - CASP is once we, we are saying, look, DeepMind is doing protein folding.
[01:31] Andrew Senior - That is how good we're, and perhaps it is higher than everyone else, perhaps it is not.
[01:37] ? - We determined to enter CASP competitors as a result of it represented the Olympics of protein folding.
[01:44] John Moult - CASP, we began to try to velocity up the answer to the protein folding drawback.
[01:50] John Moult - After we began CASP in 1994, I actually was naive about how arduous this was going to be.
[01:58] ? - It was very cumbersome to do this as a result of it took a very long time.
[02:01] ? - Let's examine what, what, what are we doing nonetheless to enhance?
[02:03] ? - Sometimes 100 completely different teams from world wide take part in CASP, and we take a set of 100 proteins and we ask the teams to ship us what they suppose the constructions appear to be.
[02:15] ? - We are able to attain 57.9 GDT on CASP 12 floor reality.
[02:19] John Jumper - CASP has a metric on which you'll be scored, which is that this GDT metric.
[02:25] John Jumper - On a scale of zero to 100, you'll anticipate a GDT over 90 to be an answer to the issue.
[02:33] ? - If we do obtain this, this has unimaginable medical relevance.
[02:37] Pushmeet Kohli - The implications are immense, from how ailments progress, how one can uncover new medicine.
[02:45] Pushmeet Kohli - It is countless.
[02:46] ? - I wished to make a, a very easy system and the outcomes have been surprisingly good.
[02:50] ? - The group acquired some outcomes with a brand new method, not solely is it extra correct, nevertheless it's a lot quicker than the previous system.
[02:56] ? - I feel we'll considerably exceed what we're doing proper now.
[02:59] ? - This can be a recreation, recreation changer, I feel.
[03:01] John Moult - In CASP 13, one thing very vital had occurred.
[03:06] John Moult - For the primary time, we noticed the efficient software of synthetic intelligence.
[03:11] ? - We have superior the cutting-edge within the subject, in order that's implausible, however we nonetheless acquired a protracted strategy to go earlier than we have solved it.
[03:18] ? - The shapes have been now roughly appropriate for most of the proteins, however the particulars, precisely the place every atom sits, which is basically what we'd name an answer, we're not but there.
[03:29] ? - It does not assist you probably have the tallest ladder when you are going to the moon.
[03:33] ? - We hit slightly little bit of a brick wall, um, since we gained CASP, then it was again to the drafting board and like what are our new concepts?
[03:41] ? - Um, after which it is taken a short time, I'd say, for them to get again to the place they have been, however with the brand new concepts.
[03:51] ? - They'll go additional, proper?
[03:52] ? - So, um, in order that's a very essential second.
[03:55] ? - I've seen that second so many occasions now, however I do know what which means now, and I do know that is the time now to press.
[04:02] ? - We have to double down and go as quick as attainable from right here.
[04:05] ? - I feel we have no time to lose.
[04:07] ? - So the intention is to enter CASP once more.
[04:09] ? - CASP is deeply aggravating.
[04:12] ? - There's one thing bizarre happening with, um, the educational as a result of it's studying one thing that is correlated with GDT, nevertheless it's not calibrated.
[04:18] ? - I really feel barely uncomfortable.
[04:20] ? - We must be studying this, you understand, within the blink of an eye fixed.
[04:23] ? - The expertise advancing outdoors DeepMind can also be doing unimaginable work.
[04:27] Richard Evans - And there is at all times the chance one other group has come someplace on the market subject that we do not even find out about.
[04:32] ? - Somebody requested me, effectively, ought to we panic now?
[04:33] ? - After all, we should always have been panicking earlier than.
[04:35] ? - It does appear to do higher, however nonetheless does not do fairly in addition to one of the best mannequin.
[04:39] ? - Um, so it appears like there's room for enchancment.
[04:42] ? - There's at all times a danger that you've got missed one thing, and that is why blind assessments like CASP are so essential to validate whether or not our outcomes are actual.
[04:49] ? - Clearly, I am excited to see how CASP 14 goes.
[04:51] ? - My expectation is we get our heads down, we give attention to the complete purpose, which is to resolve the entire drawback.
[05:14] ? - We have been ready for CASP to begin on April fifteenth as a result of that is when it was initially scheduled to begin, and it has been delayed by a month as a consequence of coronavirus.
[05:24] ? - I actually miss everybody.
[05:25] ? - No, I struggled slightly bit simply form of getting right into a routine, particularly, uh, my spouse, she got here down with the, the virus.
[05:32] ? - I imply, fortunately it did not end up too critical.
[05:34] ? - CASP began on Monday.
[05:37] Demis Hassabis - Can I simply examine this diagram you have acquired right here, John, this one the place we ask floor reality.
[05:40] Demis Hassabis - Is that this one we have finished badly on?
[05:42] ? - We're truly fairly good on this area.
[05:43] ? - If you happen to think about that we hadn't have stated it got here round this fashion, however had put it in.
[05:47] ? - Yeah, and that as an alternative.
[05:48] ? - Yeah.
[05:49] ? - One of many hardest proteins we have gotten in CASP to date is a SARS-CoV-2 protein, uh, referred to as Orf8.
[05:55] ? - Orf8 is a coronavirus protein.
[05:57] ? - We tried actually arduous to enhance our prediction, like actually, actually arduous, in all probability probably the most time that we have now ever spent on a single goal.
[06:05] ? - So we're about two-thirds of the way in which by way of CASP, and we have gotten three solutions again.
[06:11] ? - We now have a floor reality for Orf8, which is among the coronavirus proteins.
[06:17] ? - And it seems we did very well in predicting that.
[06:20] Demis Hassabis - Wonderful job, everybody, the entire group.
[06:23] Demis Hassabis - It has been an unimaginable effort.
[06:24] John Moult - Right here what we noticed in CASP 14 was a gaggle delivering atomic accuracy off the bat, basically fixing what in our world is 2 issues.
[06:34] John Moult - How do you look to search out the appropriate answer, after which how do you acknowledge you have acquired the appropriate answer once you're there?
[06:41] ? - All proper, are we, are we principally right here?
[06:46] ? - I will learn an e-mail.
[06:48] ? - Uh, I acquired this from John Moult.
[06:50] ? - Now I am going to simply learn it.
[06:51] ? - It says, John, as I anticipate you understand, your group has carried out amazingly effectively in CASP 14, each relative to different teams and in absolute mannequin accuracy.
[07:02] ? - Congratulations on this work.
[07:05] ? - It's actually excellent.
[07:07] Demis Hassabis - AlphaFold represents an enormous leap ahead that I hope will actually speed up drug discovery and assist us to higher perceive illness.
[07:13] John Moult - It is fairly mind-blowing.
[07:16] John Moult - You recognize, these outcomes have been, for me, having labored on this drawback so lengthy, after many, many stops and begins and can this ever get there, immediately this can be a answer.
[07:28] John Moult - We have solved the issue.
[07:29] John Moult - This provides you such pleasure about the way in which science works, about how one can by no means see precisely and even roughly what is going on to occur subsequent.
[07:37] John Moult - There are at all times these surprises, and that actually, as a scientist, is what retains you going.
[07:41] John Moult - What is going on to be the following shock?
------------------------------- finish of response --------------------------------

------------------------------- gemini-2.0-flash -------------------------------
Enter tokens   :        18
Output tokens  :     1,814
------------------------------ begin of response -------------------------------

## Terminology Used for Video Transcriptions

This is a breakdown of widespread terminology used when discussing video transcriptions:

*   **Transcription:** The overall time period for changing audio or video content material into textual content.
*   **Video Transcription:** Particularly refers to transcribing the audio portion of a video.
*   **Verbatim Transcription:** A transcription that features each spoken phrase, together with filler phrases ("um," "ah"), false begins, stutters, and background noises (if related).  It goals for a extremely correct illustration of the spoken phrase.
*   **Clear Verbatim Transcription (or Edited Transcription):** A transcription that removes filler phrases, false begins, and stutters to create a extra readable and polished textual content. It focuses on conveying the which means clearly.
*   **Clever Verbatim Transcription:** A center floor between verbatim and clear verbatim. It removes some filler phrases and stutters however retains components that contribute to the speaker's model or the context of the dialog.
*   **Abstract Transcription:** A short overview of the video's content material, highlighting the details.
*   **Timecoding (or Timestamping):** Including timestamps to the transcription to point when every part of textual content was spoken within the video.  That is essential for accessibility and navigation.
*   **Speaker Identification:** Figuring out and labeling completely different audio system within the video.
*   **Captioning (or Subtitling):** Creating textual content variations of the video's audio which are displayed on the display, often synchronized with the video.  Captions are sometimes used for accessibility (for deaf or hard-of-hearing viewers) and for viewers preferring to learn alongside.
*   **Closed Captions:** Captions that may be turned on or off by the viewer.
*   **Open Captions:** Captions which are completely embedded within the video and can't be turned off.
*   **SRT (SubRip Subtitle):** A standard file format for storing captions and subtitles.
*   **VTT (Video Textual content Tracks):** A extra trendy file format for captions and subtitles, typically used with HTML5 video.
*   **Burned-in Captions:** Captions which are completely embedded into the video file.
*   **Transcriber:** The particular person or software program that performs the transcription.
*   **Accuracy Fee:** The share of phrases transcribed appropriately.
*   **Turnaround Time:** The time it takes to finish the transcription.

## Typical Output Instance

As an example we have now a brief video clip of an interview. This is how several types of transcriptions may look:

**Situation:** A brief clip of an interview with a chef discussing their new restaurant.

**1. Verbatim Transcription (with Timecoding):**

```
00:00:00 --> 00:00:03 Interviewer: So, Chef, welcome to the present!
00:00:03 --> 00:00:06 Chef: Thanks for having me. It is, uh, nice to be right here.
00:00:06 --> 00:00:12 Interviewer: So, inform us slightly bit about your new restaurant, "The Golden Spoon."
00:00:12 --> 00:00:20 Chef: Effectively, um, it is, it has been a dream of mine for, like, years, you understand?  To, like, open a spot that actually focuses on, um, seasonal elements.
00:00:20 --> 00:00:25 Interviewer: And what sort of delicacies can individuals anticipate?
00:00:25 --> 00:00:32 Chef: We're doing, uh, trendy American, however with, like, an actual emphasis on, uh, native farms and, you understand, sustainable practices.
```

**2. Clear Verbatim Transcription (with Timecoding):**

```
00:00:00 --> 00:00:03 Interviewer: So, Chef, welcome to the present!
00:00:03 --> 00:00:06 Chef: Thanks for having me. It is nice to be right here.
00:00:06 --> 00:00:12 Interviewer: So, inform us slightly bit about your new restaurant, "The Golden Spoon."
00:00:12 --> 00:00:20 Chef: Effectively, it has been a dream of mine for years to open a spot that actually focuses on seasonal elements.
00:00:20 --> 00:00:25 Interviewer: And what sort of delicacies can individuals anticipate?
00:00:25 --> 00:00:32 Chef: We're doing trendy American, however with an actual emphasis on native farms and sustainable practices.
```

**3. Clever Verbatim Transcription (with Timecoding):**

```
00:00:00 --> 00:00:03 Interviewer: So, Chef, welcome to the present!
00:00:03 --> 00:00:06 Chef: Thanks for having me. It is nice to be right here.
00:00:06 --> 00:00:12 Interviewer: So, inform us slightly bit about your new restaurant, "The Golden Spoon."
00:00:12 --> 00:00:20 Chef: Effectively, it has been a dream of mine for, like, years, you understand? To open a spot that actually focuses on seasonal elements.
00:00:20 --> 00:00:25 Interviewer: And what sort of delicacies can individuals anticipate?
00:00:25 --> 00:00:32 Chef: We're doing trendy American, however with, like, an actual emphasis on native farms and, you understand, sustainable practices.
```

**4. Abstract Transcription:**

```
Chef discusses their new restaurant, "The Golden Spoon," which focuses on seasonal elements and trendy American delicacies with an emphasis on native farms and sustainable practices.
```

**5. SRT File (for Captions):**

```
1
00:00:00,000 --> 00:00:03,000
Interviewer: So, Chef, welcome to the present!

2
00:00:03,000 --> 00:00:06,000
Chef: Thanks for having me. It is nice to be right here.

3
00:00:06,000 --> 00:00:12,000
Interviewer: So, inform us slightly bit about your new restaurant, "The Golden Spoon."

4
00:00:12,000 --> 00:00:20,000
Chef: Effectively, it has been a dream of mine for years to open a spot that actually focuses on seasonal elements.

5
00:00:20,000 --> 00:00:25,000
Interviewer: And what sort of delicacies can individuals anticipate?

6
00:00:25,000 --> 00:00:32,000
Chef: We're doing trendy American, however with an actual emphasis on native farms and sustainable practices.
```

**Key Variations:**

*   **Verbatim:** Captures each element, together with imperfections in speech.
*   **Clear Verbatim:**  Focuses on readability and readability by eradicating pointless components.
*   **Clever Verbatim:** A steadiness between accuracy and readability.
*   **Abstract:** Supplies a concise overview.
*   **SRT:**  A structured file format for displaying captions synchronized with the video.

The very best sort of transcription is dependent upon the particular wants of the undertaking.  For authorized or analysis functions, a verbatim transcription is likely to be mandatory. For normal accessibility or advertising functions, a clear verbatim or clever verbatim transcription is commonly most popular.  Abstract transcriptions are helpful for rapidly understanding the content material of a video.  SRT information are important for including captions to movies.

------------------------------- finish of response --------------------------------

----------------- GDM_PODCAST_TRAILER_PT59S / gemini-2.0-flash -----------------
Enter tokens   :    16,800
Output tokens  :       635
------------------------------ begin of response -------------------------------

[
  {
    "start": "00:00",
    "text": "Do I have to call you Sir Demis now?",
    "voice": 1
  },
  {
    "start": "00:01",
    "text": "Oh, you don't. Absolutely not.",
    "voice": 2
  },
  {
    "start": "00:03",
    "text": "Welcome to Google Deep Mind the podcast with me, your host Professor Hannah Fry.",
    "voice": 1
  },
  {
    "start": "00:06",
    "text": "We want to take you to the heart of where these ideas are coming from. We want to introduce you to the people who are leading the design of our collective future.",
    "voice": 1
  },
  {
    "start": "00:19",
    "text": "Getting the safety right is probably, I'd say, one of the most important challenges of our time. I want safe and capable.",
    "voice": 3
  },
  {
    "start": "00:26",
    "text": "I want a bridge that will not collapse.",
    "voice": 3
  },
  {
    "start": "00:30",
    "text": "just give these scientists a superpower that they had not imagined earlier.",
    "voice": 4
  },
  {
    "start": "00:34",
    "text": "autonomous vehicles. It's hard to fathom that when you're working on a search engine.",
    "voice": 5
  },
  {
    "start": "00:38",
    "text": "We may see entirely new genre or entirely new forms of art come up. There may be a new word that is not music, painting, photography, movie making, and that AI will have helped us create it.",
    "voice": 6
  },
  {
    "start": "00:48",
    "text": "You really want AGI to be able to peer into the mysteries of the universe.",
    "voice": 1
  },
  {
    "start": "00:51",
    "text": "Yes, quantum mechanics, string theory, well, and the nature of reality.",
    "voice": 2
  },
  {
    "start": "00:55",
    "text": "Ow.",
    "voice": 1
  },
  {
    "start": "00:56",
    "text": "the magic of AI.",
    "voice": 6
  }
]

------------------------------- finish of response --------------------------------

----------------- GDM_PODCAST_TRAILER_PT59S / gemini-2.0-flash -----------------
Enter tokens   :    16,920
Output tokens  :       806
------------------------------ begin of response -------------------------------

{
  "task1_transcripts": [
    {
      "start": "00:00",
      "text": "Do I have to call you Sir Demis now?",
      "voice": 1
    },
    {
      "start": "00:01",
      "text": "Oh, you don't. Absolutely not.",
      "voice": 2
    },
    {
      "start": "00:04",
      "text": "Welcome to Google Deep Mind the podcast with me, your host Professor Hannah Fry.",
      "voice": 1
    },
    {
      "start": "00:06",
      "text": "We want to take you to the heart of where these ideas are coming from. We want to introduce you to the people who are leading the design of our collective future.",
      "voice": 1
    },
    {
      "start": "00:19",
      "text": "Getting the safety right is probably, I'd say, one of the most important challenges of our time. I want safe and capable.",
      "voice": 3
    },
    {
      "start": "00:26",
      "text": "I want a bridge that will not collapse.",
      "voice": 3
    },
    {
      "start": "00:30",
      "text": "That just give these scientists a superpower that they had not imagined earlier.",
      "voice": 4
    },
    {
      "start": "00:34",
      "text": "autonomous vehicles. It's hard to fathom that when you're working on a search engine.",
      "voice": 5
    },
    {
      "start": "00:38",
      "text": "We may see entirely new genre or entirely new forms of art come up. There may be a new word that is not music, painting, photography, movie making, and that AI will have helped us create it.",
      "voice": 6
    },
    {
      "start": "00:48",
      "text": "You really want AGI to be able to peer into the mysteries of the universe.",
      "voice": 1
    },
    {
      "start": "00:51",
      "text": "Yes, quantum mechanics, string theory, well, and the nature of reality.",
      "voice": 2
    },
    {
      "start": "00:55",
      "text": "Ow.",
      "voice": 1
    },
    {
      "start": "00:56",
      "text": "the magic of AI.",
      "voice": 6
    }
  ],
  "task2_speakers": [
    {
      "voice": 1,
      "name": "Professor Hannah Fry"
    },
    {
      "voice": 2,
      "name": "Demis Hassabis"
    },
    {
      "voice": 3,
      "name": "Anca Dragan"
    },
    {
      "voice": 4,
      "name": "Pushmeet Kohli"
    },
    {
      "voice": 5,
      "name": "Jeff Dean"
    },
    {
      "voice": 6,
      "name": "Douglas Eck"
    }
  ]
}

------------------------------- finish of response --------------------------------

import re

import pydantic
from google.genai.sorts import MediaResolution, ThinkingConfig

SamplingFrameRate = float

VIDEO_TRANSCRIPTION_PROMPT = """
**Job 1 - Transcripts**

- Watch the video and hear rigorously to the audio.
- Determine every distinctive voice utilizing a `voice` ID (1, 2, 3, and many others.).
- Transcribe the video's audio verbatim with voice diarization.
- Embrace the `begin` timecode ({timecode_spec}) for every speech section.

**Job 2 - Audio system**

- For every `voice` ID from Job 1, extract details about the corresponding speaker.
- Use visible and audio cues.
- If a chunk of knowledge can't be discovered, use a query mark (`?`) as the worth.
"""
NOT_FOUND = "?"


class Transcript(pydantic.BaseModel):
    begin: str
    textual content: str
    voice: int


class Speaker(pydantic.BaseModel):
    voice: int
    identify: str
    firm: str
    place: str
    role_in_video: str


class VideoTranscription(pydantic.BaseModel):
    task1_transcripts: checklist[Transcript] = pydantic.Subject(default_factory=checklist)
    task2_speakers: checklist[Speaker] = pydantic.Subject(default_factory=checklist)


def get_generate_content_config(mannequin: Mannequin, video: Video) -> GenerateContentConfig:
    media_resolution = get_media_resolution_for_video(video)
    thinking_config = get_thinking_config(mannequin)

    return GenerateContentConfig(
        temperature=DEFAULT_CONFIG.temperature,
        top_p=DEFAULT_CONFIG.top_p,
        seed=DEFAULT_CONFIG.seed,
        response_mime_type="software/json",
        response_schema=VideoTranscription,
        media_resolution=media_resolution,
        thinking_config=thinking_config,
    )


def get_video_duration(video: Video) -> timedelta | None:
    # For testing functions, video period is statically specified within the enum identify
    # Suffix (ISO 8601 based mostly): _PT[H][M][S]
    # For manufacturing,
    # - fetch durations dynamically or retailer them individually
    # - keep in mind video VideoMetadata.start_offset & VideoMetadata.end_offset
    regex = r"_PT(?:(d+)H)?(?:(d+)M)?(?:(d+)S)?$"
    if not (match := re.search(regex, video.identify)):
        print(f"⚠️ No period information in {video.identify}. Will use defaults.")
        return None

    h_str, m_str, s_str = match.teams()
    return timedelta(
        hours=int(h_str) if h_str is just not None else 0,
        minutes=int(m_str) if m_str is just not None else 0,
        seconds=int(s_str) if s_str is just not None else 0,
    )


def get_media_resolution_for_video(video: Video) -> MediaResolution | None:
    if not (video_duration := get_video_duration(video)):
        return None  # Default

    # For testing functions, that is based mostly on video period, as our brief movies are usually extra detailed
    less_than_five_minutes = video_duration < timedelta(minutes=5)
    if less_than_five_minutes:
        media_resolution = MediaResolution.MEDIA_RESOLUTION_MEDIUM
    else:
        media_resolution = MediaResolution.MEDIA_RESOLUTION_LOW

    return media_resolution


def get_sampling_frame_rate_for_video(video: Video) -> SamplingFrameRate | None:
    sampling_frame_rate = None  # Default (1 FPS for present fashions)

    # [Optional] Outline a customized FPS: 0.0 < sampling_frame_rate <= 24.0

    return sampling_frame_rate


def get_timecode_spec_for_model_and_video(mannequin: Mannequin, video: Video) -> str:
    timecode_spec = "MM:SS"  # Default

    match mannequin:
        case Mannequin.GEMINI_2_0_FLASH:  # Helps MM:SS
            move
        case Mannequin.GEMINI_2_5_FLASH | Mannequin.GEMINI_2_5_PRO:  # Help MM:SS and H:MM:SS
            period = get_video_duration(video)
            one_hour_or_more = period is just not None and timedelta(hours=1) <= period
            if one_hour_or_more:
                timecode_spec = "MM:SS or H:MM:SS"
        case _:
            assert False, "Add timecode format for brand new mannequin"

    return timecode_spec


def get_thinking_config(mannequin: Mannequin) -> ThinkingConfig | None:
    # Examples of considering configurations (Gemini 2.5 fashions)
    match mannequin:
        case Mannequin.GEMINI_2_5_FLASH:  # Pondering disabled
            return ThinkingConfig(thinking_budget=0, include_thoughts=False)
        case Mannequin.GEMINI_2_5_PRO:  # Minimal considering price range and no summarized ideas
            return ThinkingConfig(thinking_budget=128, include_thoughts=False)
        case _:
            return None  # Default


def get_video_transcription_from_response(
    response: GenerateContentResponse,
) -> VideoTranscription:
    if not isinstance(response.parsed, VideoTranscription):
        print("❌ Couldn't parse the JSON response")
        return VideoTranscription()  # Empty transcription

    return response.parsed


def get_video_transcription(
    video: Video,
    video_segment: VideoSegment | None = None,
    fps: float | None = None,
    immediate: str | None = None,
    mannequin: Mannequin | None = None,
) -> VideoTranscription:
    mannequin = mannequin or Mannequin.DEFAULT
    model_id = mannequin.worth

    fps = fps or get_sampling_frame_rate_for_video(video)
    video_part = get_video_part(video, video_segment, fps)
    if not video_part:  # Unsupported supply, return an empty transcription
        return VideoTranscription()
    if immediate is None:
        timecode_spec = get_timecode_spec_for_model_and_video(mannequin, video)
        immediate = VIDEO_TRANSCRIPTION_PROMPT.format(timecode_spec=timecode_spec)
    contents = [video_part, prompt.strip()]

    config = get_generate_content_config(mannequin, video)

    print(f" {video.identify} / {model_id} ".heart(80, "-"))
    response = None
    for try in get_retrier():
        with try:
            response = consumer.fashions.generate_content(
                mannequin=model_id,
                contents=contents,
                config=config,
            )
            display_response_info(response)

    assert isinstance(response, GenerateContentResponse)
    return get_video_transcription_from_response(response)

import itertools
from collections.abc import Callable, Iterator

from pandas import DataFrame, Sequence
from pandas.io.codecs.model import Styler
from pandas.io.codecs.style_render import CSSDict

BGCOLOR_COLUMN = "bg_color"  # Hidden column to retailer row background colours


def yield_known_speaker_color() -> Iterator[str]:
    PAL_40 = ("#669DF6", "#EE675C", "#FCC934", "#5BB974")
    PAL_30 = ("#8AB4F8", "#F28B82", "#FDD663", "#81C995")
    PAL_20 = ("#AECBFA", "#F6AEA9", "#FDE293", "#A8DAB5")
    PAL_10 = ("#D2E3FC", "#FAD2CF", "#FEEFC3", "#CEEAD6")
    PAL_05 = ("#E8F0FE", "#FCE8E6", "#FEF7E0", "#E6F4EA")
    return itertools.cycle([*PAL_40, *PAL_30, *PAL_20, *PAL_10, *PAL_05])


def yield_unknown_speaker_color() -> Iterator[str]:
    GRAYS = ["#80868B", "#9AA0A6", "#BDC1C6", "#DADCE0", "#E8EAED", "#F1F3F4"]
    return itertools.cycle(GRAYS)


def get_color_for_voice_mapping(audio system: checklist[Speaker]) -> dict[int, str]:
    known_speaker_color = yield_known_speaker_color()
    unknown_speaker_color = yield_unknown_speaker_color()

    mapping: dict[int, str] = {}
    for speaker in audio system:
        if speaker.identify != NOT_FOUND:
            coloration = subsequent(known_speaker_color)
        else:
            coloration = subsequent(unknown_speaker_color)
        mapping[speaker.voice] = coloration

    return mapping


def get_table_styler(df: DataFrame) -> Styler:
    def join_styles(types: checklist[str]) -> str:
        return ";".be part of(types)

    table_css = [
        "color: #202124",
        "background-color: #BDC1C6",
        "border: 0",
        "border-radius: 0.5rem",
        "border-spacing: 0px",
        "outline: 0.5rem solid #BDC1C6",
        "margin: 1rem 0.5rem",
    ]
    th_css = ["background-color: #E8EAED"]
    th_td_css = ["text-align:left", "padding: 0.25rem 1rem"]
    table_styles = [
        CSSDict(selector="", props=join_styles(table_css)),
        CSSDict(selector="th", props=join_styles(th_css)),
        CSSDict(selector="th,td", props=join_styles(th_td_css)),
    ]

    return df.model.set_table_styles(table_styles).conceal()


def change_row_bgcolor(row: Sequence) -> checklist[str]:
    model = f"background-color:{row[BGCOLOR_COLUMN]}"
    return [style] * len(row)


def display_table(yield_rows: Callable[[], Iterator[list[str]]]) -> None:
    knowledge = yield_rows()
    df = DataFrame(columns=subsequent(knowledge), knowledge=knowledge)
    styler = get_table_styler(df)
    styler.apply(change_row_bgcolor, axis=1)
    styler.conceal([BGCOLOR_COLUMN], axis="columns")

    html = styler.to_html()
    IPython.show.show(IPython.show.HTML(html))


def display_speakers(transcription: VideoTranscription) -> None:
    def sanitize_field(s: str, symbol_if_unknown: str) -> str:
        return symbol_if_unknown if s == NOT_FOUND else s

    def yield_rows() -> Iterator[list[str]]:
        yield ["voice", "name", "company", "position", "role_in_video", BGCOLOR_COLUMN]

        color_for_voice = get_color_for_voice_mapping(transcription.task2_speakers)
        for speaker in transcription.task2_speakers:
            yield [
                str(speaker.voice),
                sanitize_field(speaker.name, NOT_FOUND),
                sanitize_field(speaker.company, NOT_FOUND),
                sanitize_field(speaker.position, NOT_FOUND),
                sanitize_field(speaker.role_in_video, NOT_FOUND),
                color_for_voice.get(speaker.voice, "red"),
            ]

    display_markdown(f"### Audio system ({len(transcription.task2_speakers)})")
    display_table(yield_rows)


def display_transcripts(transcription: VideoTranscription) -> None:
    def yield_rows() -> Iterator[list[str]]:
        yield ["start", "speaker", "transcript", BGCOLOR_COLUMN]

        color_for_voice = get_color_for_voice_mapping(transcription.task2_speakers)
        speaker_for_voice = {
            speaker.voice: speaker for speaker in transcription.task2_speakers
        }
        previous_voice = None
        for transcript in transcription.task1_transcripts:
            current_voice = transcript.voice
            speaker_label = ""
            if speaker := speaker_for_voice.get(current_voice, None):
                if speaker.identify != NOT_FOUND:
                    speaker_label = speaker.identify
                elif speaker.place != NOT_FOUND:
                    speaker_label = f"[voice {current_voice}][{speaker.position}]"
                elif speaker.role_in_video != NOT_FOUND:
                    speaker_label = f"[voice {current_voice}][{speaker.role_in_video}]"
            if not speaker_label:
                speaker_label = f"[voice {current_voice}]"
            yield [
                transcript.start,
                speaker_label if current_voice != previous_voice else '"',
                transcript.text,
                color_for_voice.get(current_voice, "red"),
            ]
            previous_voice = current_voice

    display_markdown(f"### Transcripts ({len(transcription.task1_transcripts)})")
    display_table(yield_rows)


def transcribe_video(
    video: Video,
    video_segment: VideoSegment | None = None,
    fps: float | None = None,
    immediate: str | None = None,
    mannequin: Mannequin | None = None,
) -> None:
    display_video(video)
    transcription = get_video_transcription(video, video_segment, fps, immediate, mannequin)
    display_speakers(transcription)
    display_transcripts(transcription)