Upload 11 files

data/input/test_input.txt

@@ -0,0 +1,2 @@
+https://www.youtube.com/shorts/xaRRZKgj5aQ
+https://www.youtube.com/shorts/xK9_V9LF4PE

data/output/test_output.csv

@@ -0,0 +1,3 @@
+youtube_url,video_id,transcript
+https://www.youtube.com/shorts/xaRRZKgj5aQ,xaRRZKgj5aQ,"[{'text': 'what happens if you pull a shot of', 'start': 0.08, 'duration': 2.839}, {'text': 'espresso through an orange I thought', 'start': 1.24, 'duration': 3.119}, {'text': 'this could potentially taste pretty good', 'start': 2.919, 'duration': 2.801}, {'text': ""so let's try it out and see if it even"", 'start': 4.359, 'duration': 3.04}, {'text': 'works so I started off by grabbing a', 'start': 5.72, 'duration': 3.16}, {'text': 'small orange and cutting off a small', 'start': 7.399, 'duration': 3.001}, {'text': 'slice and then I measured out a pretty', 'start': 8.88, 'duration': 3.48}, {'text': 'typical 18 G dose of coffee but I', 'start': 10.4, 'duration': 3.279}, {'text': 'grounded a little bit coarser than', 'start': 12.36, 'duration': 2.8}, {'text': 'normal I then did all my pretty normal', 'start': 13.679, 'duration': 3.001}, {'text': 'Puck prep with the only exception being', 'start': 15.16, 'duration': 2.92}, {'text': 'I put an orange at the bottom of the', 'start': 16.68, 'duration': 3.28}, {'text': 'Porta filter now to be honest I was a', 'start': 18.08, 'duration': 3.039}, {'text': 'little worried that this orange was', 'start': 19.96, 'duration': 2.6}, {'text': 'going to completely block the flow of', 'start': 21.119, 'duration': 2.721}, {'text': 'espresso and if you look at what', 'start': 22.56, 'duration': 2.639}, {'text': 'happened to this first shot you can see', 'start': 23.84, 'duration': 2.96}, {'text': ""that's exactly what happened so I had a"", 'start': 25.199, 'duration': 3.281}, {'text': 'different idea I added a paper filter to', 'start': 26.8, 'duration': 2.92}, {'text': 'the bottom of the basket which should', 'start': 28.48, 'duration': 2.68}, {'text': 'allow the espresso to flow through a', 'start': 29.72, 'duration': 2.92}, {'text': ""little bit easier so let's see how the"", 'start': 31.16, 'duration': 2.96}, {'text': 'shot looks and more importantly see how', 'start': 32.64, 'duration': 3.439}, {'text': ""it tastes it's definitely not the"", 'start': 34.12, 'duration': 3.2}, {'text': ""prettiest looking shot but we can't"", 'start': 36.079, 'duration': 2.64}, {'text': 'dismiss it just on that but how does', 'start': 37.32, 'duration': 3.079}, {'text': 'this espresso', 'start': 38.719, 'duration': 3.761}, {'text': ""taste I'm kind of getting chocolate"", 'start': 40.399, 'duration': 5.281}, {'text': 'covered orange vibes', 'start': 42.48, 'duration': 3.2}]"
+https://www.youtube.com/shorts/xK9_V9LF4PE,xK9_V9LF4PE,"[{'text': 'here are snacks I packed for my flight', 'start': 0.199, 'duration': 4.241}, {'text': 'part 19 yesterday I packed snacks for', 'start': 1.959, 'duration': 4.44}, {'text': ""Doug and today I'm packing snacks for"", 'start': 4.44, 'duration': 3.439}, {'text': ""myself since we're going on separate"", 'start': 6.399, 'duration': 3.841}, {'text': ""trips I'll be departing around lunchtime"", 'start': 7.879, 'duration': 3.961}, {'text': 'so I wanted to make a little meal with', 'start': 10.24, 'duration': 3.8}, {'text': 'protein but also enough to fill me up', 'start': 11.84, 'duration': 3.52}, {'text': ""there's a character on my desk and I"", 'start': 14.04, 'duration': 3.28}, {'text': 'heard him say tuna Mayo so I decided to', 'start': 15.36, 'duration': 4.04}, {'text': ""make tuna mayo musui and I'm using the"", 'start': 17.32, 'duration': 3.4}, {'text': ""musui M that we're currently in"", 'start': 19.4, 'duration': 3.2}, {'text': ""production on we're launching very soon"", 'start': 20.72, 'duration': 4.16}, {'text': 'go to Janelle eats.com musubi for', 'start': 22.6, 'duration': 5.28}, {'text': 'updates I added some moabi QB mayo and', 'start': 24.88, 'duration': 5.12}, {'text': 'pickled daon before wrapping it in rice', 'start': 27.88, 'duration': 4.28}, {'text': ""and Nori for extra snacks I've been on a"", 'start': 30.0, 'duration': 4.719}, {'text': ""golden kiwi kick I'm peeling and slicing"", 'start': 32.16, 'duration': 4.84}, {'text': 'it for easier eating on the plane then', 'start': 34.719, 'duration': 3.801}, {'text': ""I'm packing the rest of the Rainer"", 'start': 37.0, 'duration': 3.239}, {'text': ""cherries so they don't go bad while"", 'start': 38.52, 'duration': 3.8}, {'text': ""we're gone of course at least one"", 'start': 40.239, 'duration': 4.121}, {'text': ""protein bar and I'm bringing this"", 'start': 42.32, 'duration': 4.919}, {'text': 'amazing Sichuan pepper chocolate brittle', 'start': 44.36, 'duration': 4.92}, {'text': ""so my family can try it I'm bringing two"", 'start': 47.239, 'duration': 4.0}, {'text': 'musu bees so that my sister can have the', 'start': 49.28, 'duration': 3.52}, {'text': ""other one just in case she didn't pack"", 'start': 51.239, 'duration': 6.241}, {'text': 'any snacks she turns to do that', 'start': 52.8, 'duration': 4.68}]"

requirements.txt

@@ -0,0 +1,3 @@
+pandas
+youtube-transcript-api
+streamlit

youtube_shorts_downloader/__init__.py

@@ -0,0 +1,4 @@
+import os
+
+base_dir = os.path.dirname(os.path.abspath(__file__))
+main_dir = os.path.dirname(base_dir)

youtube_shorts_downloader/app.py

@@ -0,0 +1,65 @@
+import streamlit as st
+import io
+import pandas as pd
+from youtube_shorts_downloader import main_dir
+from youtube_shorts_downloader.transcripts import get_batch_transcripts
+from youtube_shorts_downloader.input_output import parse_input_file, save_output
+
+
+st.title("YT Shorts Transcript Downloader")
+st.markdown(
+    "instructions: upload a text file with valid youtube urls, one per line, to fetch transcripts"
+)
+
+
+base = st.container(border=True)
+with base:
+    x, col1, col2 = st.columns([3, 20, 5])
+    with col1:
+        uploaded_file = st.file_uploader("Choose a File", type=["txt"])
+        col2, col3, col4 = st.columns([3, 2, 3])
+        with col2:
+            trans_button_val = st.button(label="fetch transcripts", type="primary")
+        with col3:
+            empty_container = st.container()
+        with col4:
+            placeholder = st.empty()
+
+
+@st.cache_data
+def convert_df(df: pd.DataFrame) -> "csv":
+    # IMPORTANT: Cache the conversion to prevent computation on every rerun
+    return df.to_csv().encode("utf-8")
+
+
+def button_logic(youtube_short_urls: list) -> None:
+    if trans_button_val:
+        batch_transcripts = get_batch_transcripts(youtube_short_urls)
+        df = pd.DataFrame(batch_transcripts)
+        converted_dv = convert_df(df)
+
+        with col4:
+            st.download_button(
+                label="Download transcripts",
+                data=converted_dv,
+                file_name="output.csv",
+                mime="text/csv",
+                disabled=False,
+                type="primary",
+            )
+
+
+# default_file_path = main_dir + "/data/input/test_input.txt"
+youtube_short_urls = []
+if uploaded_file is not None:
+    if uploaded_file.type == "text/plain":
+        from io import StringIO
+
+        stringio = StringIO(uploaded_file.read().decode("utf-8"))
+        for line in stringio:
+            youtube_short_urls.append(line.strip())
+    # else:
+    #     youtube_short_urls = parse_input_file(default_file_path)
+
+    with st.spinner(text="transcript pull in progress..."):
+        button_logic(youtube_short_urls)

youtube_shorts_downloader/input_output.py

@@ -0,0 +1,14 @@
+import pandas as pd
+
+
+def parse_input_file(input_file_path: str) -> list:
+    youtube_urls = []
+    with open(input_file_path, "r") as file:
+        for line in file:
+            youtube_urls.append(line.strip())
+    return youtube_urls
+
+
+def save_output(data: list, output_file_path: str) -> None:
+    df = pd.DataFrame(data)
+    df.to_csv(output_file_path, index=False)

youtube_shorts_downloader/transcripts.py

@@ -0,0 +1,53 @@
+import re
+from typing import List, Dict
+from youtube_transcript_api import YouTubeTranscriptApi
+
+
+def is_valid_youtube_shorts_url(potential_url: str) -> bool:
+    pattern = r"^https://www\.youtube\.com/shorts/[A-Za-z0-9_-]{11}$"  # youtube vido ids are always 11 chars long
+    return re.match(pattern, potential_url) is not None
+
+
+def get_single_transcript(youtube_url: str) -> dict:
+    try:
+        if is_valid_youtube_shorts_url(youtube_url):
+            video_id = youtube_url.split("/")[-1]
+            video_transcript = YouTubeTranscriptApi.get_transcript(video_id)
+            entry = {}
+            entry["youtube_url"] = youtube_url
+            entry["video_id"] = video_id
+            entry["transcript"] = video_transcript
+            return entry
+        else:
+            print(f"FAILURE: youtube_url is not valid - {youtube_url}")
+            return {}
+    except Exception as e:
+        print(
+            f"FAILURE: transcript pull for youtube_url - {youtube_url} - failed with exception {e}"
+        )
+        return {}
+
+
+def get_batch_transcripts(youtube_urls: List[str]) -> List[Dict]:
+    valid_urls = []
+    valid_vids = []
+    for i, url in enumerate(youtube_urls):
+        if is_valid_youtube_shorts_url(url):
+            vid = url.split("/")[-1]
+            valid_urls.append(url)
+            valid_vids.append(vid)
+    try:
+        video_transcripts = YouTubeTranscriptApi.get_transcripts(
+            valid_vids, languages=["en"]
+        )[0]
+        entries = []
+        for i in range(len(valid_urls)):
+            entry = {}
+            entry["youtube_url"] = valid_urls[i]
+            entry["video_id"] = valid_vids[i]
+            entry["transcript"] = video_transcripts[valid_vids[i]]
+            entries.append(entry)
+        return entries
+    except Exception as e:
+        print(f"FAILURE: batch transcription fetch failed with exception {e}")
+        return []