Revamp of unit tests (#775)

* Updated test suite to use new more robust mock generator.
* Fixed flake8 issues.
* Added type annotations and return to mock generating function.
* Added exception test to get_ytplayer_js.
* Added test for uniqueify helper function.
* Updated README to reference YouTube Rewind 2019 instead of gangnam style.
* Added cached HTML for various URLs associated with the youtube rewind 2019 video, in order to prevent a unit test from running in to 429 responses.
* Removed url parameter that was breaking get_video_info response.
* Fixed unit test to match change to extract implementation.
* Added additional pattern for fetching js url.
* Changed fixture generation to be based on strictly HTML, rather than partial serialization of YouTube object.

README.md

@@ -32,8 +32,8 @@ Finally *pytube* also includes a command-line utility, allowing you to quickly d
 ### Behold, a perfect balance of simplicity versus flexibility:
 
 ```python
- >>> YouTube('https://youtu.be/9bZkp7q19f0').streams.first().download()
- >>> yt = YouTube('http://youtube.com/watch?v=9bZkp7q19f0')
+ >>> YouTube('https://youtu.be/2lAe1cqCOXo').streams.first().download()
+ >>> yt = YouTube('http://youtube.com/watch?v=2lAe1cqCOXo')
  >>> yt.streams
   ... .filter(progressive=True, file_extension='mp4')
   ... .order_by('resolution')
@@ -67,37 +67,39 @@ Let's begin with showing how easy it is to download a video with pytube:
 
 ```python
 >>> from pytube import YouTube
->>> YouTube('http://youtube.com/watch?v=9bZkp7q19f0').streams.first().download()
+>>> YouTube('https://youtube.com/watch?v=2lAe1cqCOXo').streams.first().download()
 ```
 This example will download the highest quality progressive download stream available.
 
 Next, let's explore how we would view what video streams are available:
 
 ```python
->>> yt = YouTube('http://youtube.com/watch?v=9bZkp7q19f0')
->>> yt.streams.all()
- [<Stream: itag="22" mime_type="video/mp4" res="720p" fps="30fps" vcodec="avc1.64001F" acodec="mp4a.40.2">,
- <Stream: itag="43" mime_type="video/webm" res="360p" fps="30fps" vcodec="vp8.0" acodec="vorbis">,
- <Stream: itag="18" mime_type="video/mp4" res="360p" fps="30fps" vcodec="avc1.42001E" acodec="mp4a.40.2">,
- <Stream: itag="36" mime_type="video/3gpp" res="240p" fps="30fps" vcodec="mp4v.20.3" acodec="mp4a.40.2">,
- <Stream: itag="17" mime_type="video/3gpp" res="144p" fps="30fps" vcodec="mp4v.20.3" acodec="mp4a.40.2">,
- <Stream: itag="137" mime_type="video/mp4" res="1080p" fps="30fps" vcodec="avc1.640028">,
- <Stream: itag="248" mime_type="video/webm" res="1080p" fps="30fps" vcodec="vp9">,
- <Stream: itag="136" mime_type="video/mp4" res="720p" fps="30fps" vcodec="avc1.4d401f">,
- <Stream: itag="247" mime_type="video/webm" res="720p" fps="30fps" vcodec="vp9">,
- <Stream: itag="135" mime_type="video/mp4" res="480p" fps="30fps" vcodec="avc1.4d401e">,
- <Stream: itag="244" mime_type="video/webm" res="480p" fps="30fps" vcodec="vp9">,
- <Stream: itag="134" mime_type="video/mp4" res="360p" fps="30fps" vcodec="avc1.4d401e">,
- <Stream: itag="243" mime_type="video/webm" res="360p" fps="30fps" vcodec="vp9">,
- <Stream: itag="133" mime_type="video/mp4" res="240p" fps="30fps" vcodec="avc1.4d4015">,
- <Stream: itag="242" mime_type="video/webm" res="240p" fps="30fps" vcodec="vp9">,
- <Stream: itag="160" mime_type="video/mp4" res="144p" fps="30fps" vcodec="avc1.4d400c">,
- <Stream: itag="278" mime_type="video/webm" res="144p" fps="30fps" vcodec="vp9">,
- <Stream: itag="140" mime_type="audio/mp4" abr="128kbps" acodec="mp4a.40.2">,
- <Stream: itag="171" mime_type="audio/webm" abr="128kbps" acodec="vorbis">,
- <Stream: itag="249" mime_type="audio/webm" abr="50kbps" acodec="opus">,
- <Stream: itag="250" mime_type="audio/webm" abr="70kbps" acodec="opus">,
- <Stream: itag="251" mime_type="audio/webm" abr="160kbps" acodec="opus">]
+>>> yt = YouTube('https://youtube.com/watch?v=2lAe1cqCOXo')
+>>> yt.streams
+ [<Stream: itag="18" mime_type="video/mp4" res="360p" fps="30fps" vcodec="avc1.42001E" acodec="mp4a.40.2" progressive="True" type="video">,
+ <Stream: itag="22" mime_type="video/mp4" res="720p" fps="30fps" vcodec="avc1.64001F" acodec="mp4a.40.2" progressive="True" type="video">,
+ <Stream: itag="137" mime_type="video/mp4" res="1080p" fps="30fps" vcodec="avc1.640028" progressive="False" type="video">,
+ <Stream: itag="248" mime_type="video/webm" res="1080p" fps="30fps" vcodec="vp9" progressive="False" type="video">,
+ <Stream: itag="399" mime_type="video/mp4" res="None" fps="30fps" vcodec="av01.0.08M.08" progressive="False" type="video">,
+ <Stream: itag="136" mime_type="video/mp4" res="720p" fps="30fps" vcodec="avc1.4d401f" progressive="False" type="video">,
+ <Stream: itag="247" mime_type="video/webm" res="720p" fps="30fps" vcodec="vp9" progressive="False" type="video">,
+ <Stream: itag="398" mime_type="video/mp4" res="None" fps="30fps" vcodec="av01.0.05M.08" progressive="False" type="video">,
+ <Stream: itag="135" mime_type="video/mp4" res="480p" fps="30fps" vcodec="avc1.4d401e" progressive="False" type="video">,
+ <Stream: itag="244" mime_type="video/webm" res="480p" fps="30fps" vcodec="vp9" progressive="False" type="video">,
+ <Stream: itag="397" mime_type="video/mp4" res="None" fps="30fps" vcodec="av01.0.04M.08" progressive="False" type="video">,
+ <Stream: itag="134" mime_type="video/mp4" res="360p" fps="30fps" vcodec="avc1.4d401e" progressive="False" type="video">,
+ <Stream: itag="243" mime_type="video/webm" res="360p" fps="30fps" vcodec="vp9" progressive="False" type="video">,
+ <Stream: itag="396" mime_type="video/mp4" res="None" fps="30fps" vcodec="av01.0.01M.08" progressive="False" type="video">,
+ <Stream: itag="133" mime_type="video/mp4" res="240p" fps="30fps" vcodec="avc1.4d4015" progressive="False" type="video">,
+ <Stream: itag="242" mime_type="video/webm" res="240p" fps="30fps" vcodec="vp9" progressive="False" type="video">,
+ <Stream: itag="395" mime_type="video/mp4" res="None" fps="30fps" vcodec="av01.0.00M.08" progressive="False" type="video">,
+ <Stream: itag="160" mime_type="video/mp4" res="144p" fps="30fps" vcodec="avc1.4d400c" progressive="False" type="video">,
+ <Stream: itag="278" mime_type="video/webm" res="144p" fps="30fps" vcodec="vp9" progressive="False" type="video">,
+ <Stream: itag="394" mime_type="video/mp4" res="None" fps="30fps" vcodec="av01.0.00M.08" progressive="False" type="video">,
+ <Stream: itag="140" mime_type="audio/mp4" abr="128kbps" acodec="mp4a.40.2" progressive="False" type="audio">,
+ <Stream: itag="249" mime_type="audio/webm" abr="50kbps" acodec="opus" progressive="False" type="audio">,
+ <Stream: itag="250" mime_type="audio/webm" abr="70kbps" acodec="opus" progressive="False" type="audio">,
+ <Stream: itag="251" mime_type="audio/webm" abr="160kbps" acodec="opus" progressive="False" type="audio">]
 ```
 You may notice that some streams listed have both a video codec and audio codec, while others have just video or just audio, this is a result of YouTube supporting a streaming technique called Dynamic Adaptive Streaming over HTTP (DASH).
 
@@ -108,35 +110,37 @@ The legacy streams that contain the audio and video in a single file (referred t
 To only view these progressive download streams:
 
 ```python
- >>> yt.streams.filter(progressive=True).all()
-  [<Stream: itag="22" mime_type="video/mp4" res="720p" fps="30fps" vcodec="avc1.64001F" acodec="mp4a.40.2">,
-  <Stream: itag="43" mime_type="video/webm" res="360p" fps="30fps" vcodec="vp8.0" acodec="vorbis">,
-  <Stream: itag="18" mime_type="video/mp4" res="360p" fps="30fps" vcodec="avc1.42001E" acodec="mp4a.40.2">,
-  <Stream: itag="36" mime_type="video/3gpp" res="240p" fps="30fps" vcodec="mp4v.20.3" acodec="mp4a.40.2">,
-  <Stream: itag="17" mime_type="video/3gpp" res="144p" fps="30fps" vcodec="mp4v.20.3" acodec="mp4a.40.2">]
+ >>> yt.streams.filter(progressive=True)
+  [<Stream: itag="18" mime_type="video/mp4" res="360p" fps="30fps" vcodec="avc1.42001E" acodec="mp4a.40.2" progressive="True" type="video">,
+  <Stream: itag="22" mime_type="video/mp4" res="720p" fps="30fps" vcodec="avc1.64001F" acodec="mp4a.40.2" progressive="True" type="video">]
 ```
 
 Conversely, if you only want to see the DASH streams (also referred to as "adaptive") you can do:
 
 ```python
->>> yt.streams.filter(adaptive=True).all()
- [<Stream: itag="137" mime_type="video/mp4" res="1080p" fps="30fps" vcodec="avc1.640028">,
-  <Stream: itag="248" mime_type="video/webm" res="1080p" fps="30fps" vcodec="vp9">,
-  <Stream: itag="136" mime_type="video/mp4" res="720p" fps="30fps" vcodec="avc1.4d401f">,
-  <Stream: itag="247" mime_type="video/webm" res="720p" fps="30fps" vcodec="vp9">,
-  <Stream: itag="135" mime_type="video/mp4" res="480p" fps="30fps" vcodec="avc1.4d401e">,
-  <Stream: itag="244" mime_type="video/webm" res="480p" fps="30fps" vcodec="vp9">,
-  <Stream: itag="134" mime_type="video/mp4" res="360p" fps="30fps" vcodec="avc1.4d401e">,
-  <Stream: itag="243" mime_type="video/webm" res="360p" fps="30fps" vcodec="vp9">,
-  <Stream: itag="133" mime_type="video/mp4" res="240p" fps="30fps" vcodec="avc1.4d4015">,
-  <Stream: itag="242" mime_type="video/webm" res="240p" fps="30fps" vcodec="vp9">,
-  <Stream: itag="160" mime_type="video/mp4" res="144p" fps="30fps" vcodec="avc1.4d400c">,
-  <Stream: itag="278" mime_type="video/webm" res="144p" fps="30fps" vcodec="vp9">,
-  <Stream: itag="140" mime_type="audio/mp4" abr="128kbps" acodec="mp4a.40.2">,
-  <Stream: itag="171" mime_type="audio/webm" abr="128kbps" acodec="vorbis">,
-  <Stream: itag="249" mime_type="audio/webm" abr="50kbps" acodec="opus">,
-  <Stream: itag="250" mime_type="audio/webm" abr="70kbps" acodec="opus">,
-  <Stream: itag="251" mime_type="audio/webm" abr="160kbps" acodec="opus">]
+>>> yt.streams.filter(adaptive=True)
+ [<Stream: itag="137" mime_type="video/mp4" res="1080p" fps="30fps" vcodec="avc1.640028" progressive="False" type="video">,
+ <Stream: itag="248" mime_type="video/webm" res="1080p" fps="30fps" vcodec="vp9" progressive="False" type="video">,
+ <Stream: itag="399" mime_type="video/mp4" res="None" fps="30fps" vcodec="av01.0.08M.08" progressive="False" type="video">,
+ <Stream: itag="136" mime_type="video/mp4" res="720p" fps="30fps" vcodec="avc1.4d401f" progressive="False" type="video">,
+ <Stream: itag="247" mime_type="video/webm" res="720p" fps="30fps" vcodec="vp9" progressive="False" type="video">,
+ <Stream: itag="398" mime_type="video/mp4" res="None" fps="30fps" vcodec="av01.0.05M.08" progressive="False" type="video">,
+ <Stream: itag="135" mime_type="video/mp4" res="480p" fps="30fps" vcodec="avc1.4d401e" progressive="False" type="video">,
+ <Stream: itag="244" mime_type="video/webm" res="480p" fps="30fps" vcodec="vp9" progressive="False" type="video">,
+ <Stream: itag="397" mime_type="video/mp4" res="None" fps="30fps" vcodec="av01.0.04M.08" progressive="False" type="video">,
+ <Stream: itag="134" mime_type="video/mp4" res="360p" fps="30fps" vcodec="avc1.4d401e" progressive="False" type="video">,
+ <Stream: itag="243" mime_type="video/webm" res="360p" fps="30fps" vcodec="vp9" progressive="False" type="video">,
+ <Stream: itag="396" mime_type="video/mp4" res="None" fps="30fps" vcodec="av01.0.01M.08" progressive="False" type="video">,
+ <Stream: itag="133" mime_type="video/mp4" res="240p" fps="30fps" vcodec="avc1.4d4015" progressive="False" type="video">,
+ <Stream: itag="242" mime_type="video/webm" res="240p" fps="30fps" vcodec="vp9" progressive="False" type="video">,
+ <Stream: itag="395" mime_type="video/mp4" res="None" fps="30fps" vcodec="av01.0.00M.08" progressive="False" type="video">,
+ <Stream: itag="160" mime_type="video/mp4" res="144p" fps="30fps" vcodec="avc1.4d400c" progressive="False" type="video">,
+ <Stream: itag="278" mime_type="video/webm" res="144p" fps="30fps" vcodec="vp9" progressive="False" type="video">,
+ <Stream: itag="394" mime_type="video/mp4" res="None" fps="30fps" vcodec="av01.0.00M.08" progressive="False" type="video">,
+ <Stream: itag="140" mime_type="audio/mp4" abr="128kbps" acodec="mp4a.40.2" progressive="False" type="audio">,
+ <Stream: itag="249" mime_type="audio/webm" abr="50kbps" acodec="opus" progressive="False" type="audio">,
+ <Stream: itag="250" mime_type="audio/webm" abr="70kbps" acodec="opus" progressive="False" type="audio">,
+ <Stream: itag="251" mime_type="audio/webm" abr="160kbps" acodec="opus" progressive="False" type="audio">]
 ```
 
 You can also download a complete Youtube playlist:
@@ -156,27 +160,32 @@ Pytube allows you to filter on every property available (see the documentation f
 To list the audio only streams:
 
 ```python
->>> yt.streams.filter(only_audio=True).all()
-  [<Stream: itag="140" mime_type="audio/mp4" abr="128kbps" acodec="mp4a.40.2">,
-  <Stream: itag="171" mime_type="audio/webm" abr="128kbps" acodec="vorbis">,
-  <Stream: itag="249" mime_type="audio/webm" abr="50kbps" acodec="opus">,
-  <Stream: itag="250" mime_type="audio/webm" abr="70kbps" acodec="opus">,
-  <Stream: itag="251" mime_type="audio/webm" abr="160kbps" acodec="opus">]
+>>> yt.streams.filter(only_audio=True)
+  [<Stream: itag="140" mime_type="audio/mp4" abr="128kbps" acodec="mp4a.40.2" progressive="False" type="audio">,
+  <Stream: itag="249" mime_type="audio/webm" abr="50kbps" acodec="opus" progressive="False" type="audio">,
+  <Stream: itag="250" mime_type="audio/webm" abr="70kbps" acodec="opus" progressive="False" type="audio">,
+  <Stream: itag="251" mime_type="audio/webm" abr="160kbps" acodec="opus" progressive="False" type="audio">]
 ```
 
 To list only ``mp4`` streams:
 
 ```python
 >>> yt.streams.filter(subtype='mp4').all()
- [<Stream: itag="22" mime_type="video/mp4" res="720p" fps="30fps" vcodec="avc1.64001F" acodec="mp4a.40.2">,
-  <Stream: itag="18" mime_type="video/mp4" res="360p" fps="30fps" vcodec="avc1.42001E" acodec="mp4a.40.2">,
-  <Stream: itag="137" mime_type="video/mp4" res="1080p" fps="30fps" vcodec="avc1.640028">,
-  <Stream: itag="136" mime_type="video/mp4" res="720p" fps="30fps" vcodec="avc1.4d401f">,
-  <Stream: itag="135" mime_type="video/mp4" res="480p" fps="30fps" vcodec="avc1.4d401e">,
-  <Stream: itag="134" mime_type="video/mp4" res="360p" fps="30fps" vcodec="avc1.4d401e">,
-  <Stream: itag="133" mime_type="video/mp4" res="240p" fps="30fps" vcodec="avc1.4d4015">,
-  <Stream: itag="160" mime_type="video/mp4" res="144p" fps="30fps" vcodec="avc1.4d400c">,
-  <Stream: itag="140" mime_type="audio/mp4" abr="128kbps" acodec="mp4a.40.2">]
+ [<Stream: itag="18" mime_type="video/mp4" res="360p" fps="30fps" vcodec="avc1.42001E" acodec="mp4a.40.2" progressive="True" type="video">,
+ <Stream: itag="22" mime_type="video/mp4" res="720p" fps="30fps" vcodec="avc1.64001F" acodec="mp4a.40.2" progressive="True" type="video">,
+ <Stream: itag="137" mime_type="video/mp4" res="1080p" fps="30fps" vcodec="avc1.640028" progressive="False" type="video">,
+ <Stream: itag="399" mime_type="video/mp4" res="None" fps="30fps" vcodec="av01.0.08M.08" progressive="False" type="video">,
+ <Stream: itag="136" mime_type="video/mp4" res="720p" fps="30fps" vcodec="avc1.4d401f" progressive="False" type="video">,
+ <Stream: itag="398" mime_type="video/mp4" res="None" fps="30fps" vcodec="av01.0.05M.08" progressive="False" type="video">,
+ <Stream: itag="135" mime_type="video/mp4" res="480p" fps="30fps" vcodec="avc1.4d401e" progressive="False" type="video">,
+ <Stream: itag="397" mime_type="video/mp4" res="None" fps="30fps" vcodec="av01.0.04M.08" progressive="False" type="video">,
+ <Stream: itag="134" mime_type="video/mp4" res="360p" fps="30fps" vcodec="avc1.4d401e" progressive="False" type="video">,
+ <Stream: itag="396" mime_type="video/mp4" res="None" fps="30fps" vcodec="av01.0.01M.08" progressive="False" type="video">,
+ <Stream: itag="133" mime_type="video/mp4" res="240p" fps="30fps" vcodec="avc1.4d4015" progressive="False" type="video">,
+ <Stream: itag="395" mime_type="video/mp4" res="None" fps="30fps" vcodec="av01.0.00M.08" progressive="False" type="video">,
+ <Stream: itag="160" mime_type="video/mp4" res="144p" fps="30fps" vcodec="avc1.4d400c" progressive="False" type="video">,
+ <Stream: itag="394" mime_type="video/mp4" res="None" fps="30fps" vcodec="av01.0.00M.08" progressive="False" type="video">,
+ <Stream: itag="140" mime_type="audio/mp4" abr="128kbps" acodec="mp4a.40.2" progressive="False" type="audio">]
 ```
 
 Multiple filters can also be specified:
@@ -185,20 +194,22 @@ Multiple filters can also be specified:
 >>> yt.streams.filter(subtype='mp4', progressive=True).all()
 >>> # this can also be expressed as:
 >>> yt.streams.filter(subtype='mp4').filter(progressive=True).all()
-  [<Stream: itag="22" mime_type="video/mp4" res="720p" fps="30fps" vcodec="avc1.64001F" acodec="mp4a.40.2">,
-  <Stream: itag="18" mime_type="video/mp4" res="360p" fps="30fps" vcodec="avc1.42001E" acodec="mp4a.40.2">]
+  [<Stream: itag="18" mime_type="video/mp4" res="360p" fps="30fps" vcodec="avc1.42001E" acodec="mp4a.40.2" progressive="True" type="video">,
+  <Stream: itag="22" mime_type="video/mp4" res="720p" fps="30fps" vcodec="avc1.64001F" acodec="mp4a.40.2" progressive="True" type="video">]
 ```
 You also have an interface to select streams by their itag, without needing to filter:
 
 ```python
 >>> yt.streams.get_by_itag(22)
-  <Stream: itag="22" mime_type="video/mp4" res="720p" fps="30fps" vcodec="avc1.64001F" acodec="mp4a.40.2">
+  <Stream: itag="22" mime_type="video/mp4" res="720p" fps="30fps" vcodec="avc1.64001F" acodec="mp4a.40.2" progressive="True" type="video">
 ```
 
 If you need to optimize for a specific feature, such as the "highest resolution" or "lowest average bitrate":
 
 ```python
 >>> yt.streams.filter(progressive=True).order_by('resolution').desc().all()
+  [<Stream: itag="22" mime_type="video/mp4" res="720p" fps="30fps" vcodec="avc1.64001F" acodec="mp4a.40.2" progressive="True" type="video">,
+  <Stream: itag="18" mime_type="video/mp4" res="360p" fps="30fps" vcodec="avc1.42001E" acodec="mp4a.40.2" progressive="True" type="video">]
 ```
 Note that ``order_by`` cannot be used if your attribute is undefined in any of the Stream instances, so be sure to apply a filter to remove those before calling it.
 
@@ -227,12 +238,12 @@ pytube also ships with a tiny cli interface for downloading and probing videos.
 Let's start with downloading:
 
 ```bash
-$ pytube http://youtube.com/watch?v=9bZkp7q19f0 --itag=22
+$ pytube http://youtube.com/watch?v=2lAe1cqCOXo --itag=22
 ```
 To view available streams:
 
 ```bash
-$ pytube http://youtube.com/watch?v=9bZkp7q19f0 --list
+$ pytube http://youtube.com/watch?v=2lAe1cqCOXo --list
 ```
 
 Finally, if you're filing a bug report, the cli contains a switch called ``--build-playback-report``, which bundles up the state, allowing others to easily replay your issue.

pytube/__main__.py

@@ -9,7 +9,6 @@ smaller peripheral modules and functions.
 """
 import json
 import logging
-from html import unescape
 from typing import Dict
 from typing import List
 from typing import Optional
@@ -113,27 +112,17 @@ class YouTube:
         :rtype: None
 
         """
-        logger.info("init started")
-
         self.vid_info = dict(parse_qsl(self.vid_info_raw))
-        if self.age_restricted:
-            self.player_config_args = self.vid_info
-        else:
-            assert self.watch_html is not None
-            self.player_config_args = get_ytplayer_config(self.watch_html)[
-                "args"
-            ]
-
-            # Fix for KeyError: 'title' issue #434
-            if "title" not in self.player_config_args:  # type: ignore
-                i_start = self.watch_html.lower().index("<title>") + len(
-                    "<title>"
-                )
-                i_end = self.watch_html.lower().index("</title>")
-                title = self.watch_html[i_start:i_end].strip()
-                index = title.lower().rfind(" - youtube")
-                title = title[:index] if index > 0 else title
-                self.player_config_args["title"] = unescape(title)
+        self.player_config_args = self.vid_info
+        self.player_response = json.loads(self.vid_info['player_response'])
+
+        # On pre-signed videos, we need to use get_ytplayer_config to fix
+        #  the player_response item
+        if 'streamingData' not in self.player_config_args['player_response']:
+            config_response = get_ytplayer_config(
+                self.watch_html
+            )['args']['player_response']
+            self.player_config_args['player_response'] = config_response
 
         # https://github.com/nficano/pytube/issues/165
         stream_maps = ["url_encoded_fmt_stream_map"]
@@ -165,8 +154,6 @@ class YouTube:
         self.stream_monostate.title = self.title
         self.stream_monostate.duration = self.length
 
-        logger.info("init finished successfully")
-
     def prefetch(self) -> None:
         """Eagerly download all necessary data.
 
@@ -280,9 +267,7 @@ class YouTube:
         :rtype: str
 
         """
-        return self.player_config_args.get("title") or (
-            self.player_response.get("videoDetails", {}).get("title")
-        )
+        return self.player_response['videoDetails']['title']
 
     @property
     def description(self) -> str:

pytube/extract.py

@@ -71,7 +71,6 @@ def video_info_url(video_id: str, watch_url: str) -> str:
     params = OrderedDict(
         [
             ("video_id", video_id),
-            ("el", "$el"),
             ("ps", "default"),
             ("eurl", quote(watch_url)),
             ("hl", "en_US"),
@@ -160,6 +159,7 @@ def get_ytplayer_js(html: str) -> Any:
     """
     js_url_patterns = [
         r"\"jsUrl\":\"([^\"]*)\"",
+        r"\"js\":\"([^\"]*base\.js)\""
     ]
     for pattern in js_url_patterns:
         regex = re.compile(pattern)

pytube/helpers.py

@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 """Various helper functions implemented by pytube."""
 import functools
+import gzip
+import json
 import logging
 import os
 import re
@@ -174,3 +176,44 @@ def uniqueify(duped_list: List) -> List:
         seen[item] = True
         result.append(item)
     return result
+
+
+def create_mock_html_json(vid_id) -> Dict[str, Any]:
+    """Generate a json.gz file with sample html responses.
+
+    :param str vid_id
+        YouTube video id
+
+    :return dict data
+        Dict used to generate the json.gz file
+    """
+    from pytube import YouTube
+    gzip_filename = 'yt-video-%s-html.json.gz' % vid_id
+
+    # Get the pytube directory in order to navigate to /tests/mocks
+    pytube_dir_path = os.path.abspath(
+        os.path.join(
+            os.path.dirname(__file__),
+            os.path.pardir
+        )
+    )
+    pytube_mocks_path = os.path.join(pytube_dir_path, 'tests', 'mocks')
+    gzip_filepath = os.path.join(pytube_mocks_path, gzip_filename)
+
+    yt = YouTube(
+        'https://www.youtube.com/watch?v=%s' % vid_id,
+        defer_prefetch_init=True
+    )
+    yt.prefetch()
+    html_data = {
+        'url': yt.watch_url,
+        'js': yt.js,
+        'embed_html': yt.embed_html,
+        'watch_html': yt.watch_html,
+        'vid_info_raw': yt.vid_info_raw
+    }
+
+    with gzip.open(gzip_filepath, 'wb') as f:
+        f.write(json.dumps(html_data).encode('utf-8'))
+
+    return html_data

tests/conftest.py

@@ -3,6 +3,7 @@
 import gzip
 import json
 import os
+from unittest import mock
 
 import pytest
 
@@ -19,35 +20,41 @@ def load_playback_file(filename):
         return json.loads(content)
 
 
-def load_and_init_from_playback_file(filename):
+@mock.patch('pytube.request.urlopen')
+def load_and_init_from_playback_file(filename, mock_urlopen):
     """Load a gzip json playback file and create YouTube instance."""
     pb = load_playback_file(filename)
-    yt = YouTube(pb["url"], defer_prefetch_init=True)
-    yt.watch_html = pb["watch_html"]
-    yt.js = pb["js"]
-    yt.vid_info = pb["video_info"]
-    yt.descramble()
-    return yt
+
+    # Mock the responses to YouTube
+    mock_url_open_object = mock.Mock()
+    mock_url_open_object.read.side_effect = [
+        pb['watch_html'].encode('utf-8'),
+        pb['vid_info_raw'].encode('utf-8'),
+        pb['js'].encode('utf-8')
+    ]
+    mock_urlopen.return_value = mock_url_open_object
+
+    return YouTube(pb["url"])
 
 
 @pytest.fixture
 def cipher_signature():
-    """Youtube instance initialized with video id 9bZkp7q19f0."""
-    filename = "yt-video-9bZkp7q19f0.json.gz"
+    """Youtube instance initialized with video id 2lAe1cqCOXo."""
+    filename = "yt-video-2lAe1cqCOXo-html.json.gz"
     return load_and_init_from_playback_file(filename)
 
 
 @pytest.fixture
 def presigned_video():
     """Youtube instance initialized with video id QRS8MkLhQmM."""
-    filename = "yt-video-QRS8MkLhQmM.json.gz"
+    filename = "yt-video-QRS8MkLhQmM-html.json.gz"
     return load_and_init_from_playback_file(filename)
 
 
 @pytest.fixture
 def age_restricted():
-    """Youtube instance initialized with video id zRbsm3e2ltw."""
-    filename = "yt-video-irauhITDrsE.json.gz"
+    """Youtube instance initialized with video id irauhITDrsE."""
+    filename = "yt-video-irauhITDrsE-html.json.gz"
     return load_playback_file(filename)
 
 
@@ -83,8 +90,8 @@ def stream_dict():
     file_path = os.path.join(
         os.path.dirname(os.path.realpath(__file__)),
         "mocks",
-        "yt-video-WXxV9g7lsFE.json.gz",
+        "yt-video-WXxV9g7lsFE-html.json.gz",
     )
     with gzip.open(file_path, "rb") as f:
-        content = f.read().decode("utf-8")
-        return json.loads(content)
+        content = json.loads(f.read().decode("utf-8"))
+        return content['watch_html']

tests/generate_fixture.py

@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-# flake8: noqa: E402
-import json
-import sys
-from os import path
-
-from pytube import YouTube
-
-currentdir = path.dirname(path.realpath(__file__))
-parentdir = path.dirname(currentdir)
-sys.path.append(parentdir)
-
-
-yt = YouTube(sys.argv[1], defer_prefetch_init=True)
-yt.prefetch()
-output = {
-    "url": sys.argv[1],
-    "watch_html": yt.watch_html,
-    "video_info": yt.vid_info,
-    "js": yt.js,
-    "embed_html": yt.embed_html,
-}
-
-outpath = path.join(currentdir, "mocks", "yt-video-" + yt.video_id + ".json")
-print("Writing to: " + outpath)
-with open(outpath, "w") as f:
-    json.dump(output, f)

tests/test_cli.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 import argparse
+import os
 from unittest import mock
 from unittest.mock import MagicMock
 from unittest.mock import patch
@@ -472,12 +473,12 @@ def test_ffmpeg_downloader(unique_name, download, run, unlink):
         [
             "ffmpeg",
             "-i",
-            "target/video_name.video_subtype",
+            os.path.join("target", "video_name.video_subtype"),
             "-i",
-            "target/audio_name.audio_subtype",
+            os.path.join("target", "audio_name.audio_subtype"),
             "-codec",
             "copy",
-            "target/safe_title.video_subtype",
+            os.path.join("target", "safe_title.video_subtype"),
         ]
     )
     unlink.assert_called()

tests/test_extract.py

@@ -7,9 +7,9 @@ from pytube.exceptions import RegexMatchError
 
 
 def test_extract_video_id():
-    url = "https://www.youtube.com/watch?v=9bZkp7q19f0"
+    url = "https://www.youtube.com/watch?v=2lAe1cqCOXo"
     video_id = extract.video_id(url)
-    assert video_id == "9bZkp7q19f0"
+    assert video_id == "2lAe1cqCOXo"
 
 
 def test_info_url(age_restricted):
@@ -29,16 +29,16 @@ def test_info_url_age_restricted(cipher_signature):
         watch_url=cipher_signature.watch_url,
     )
     expected = (
-        "https://youtube.com/get_video_info?video_id=9bZkp7q19f0&el=%24el"
+        "https://youtube.com/get_video_info?video_id=2lAe1cqCOXo"
         "&ps=default&eurl=https%253A%2F%2Fyoutube.com%2Fwatch%253Fv%"
-        "253D9bZkp7q19f0&hl=en_US"
+        "253D2lAe1cqCOXo&hl=en_US"
     )
     assert video_info_url == expected
 
 
 def test_js_url(cipher_signature):
     expected = (
-        "https://youtube.com/s/player/4a1799bd/player_ias.vflset/en_US/base.js"
+        "https://youtube.com/s/player/9b65e980/player_ias.vflset/en_US/base.js"
     )
     result = extract.js_url(cipher_signature.watch_html)
     assert expected == result
@@ -70,6 +70,12 @@ def test_get_ytplayer_config_with_no_match_should_error():
         extract.get_ytplayer_config("")
 
 
+def test_get_ytplayer_js_with_no_match_should_error():
+    with pytest.raises(RegexMatchError):
+        extract.get_ytplayer_js("")
+
+
 def test_signature_cipher_does_not_error(stream_dict):
-    extract.apply_descrambler(stream_dict, "url_encoded_fmt_stream_map")
-    assert "s" in stream_dict["url_encoded_fmt_stream_map"][0].keys()
+    config_args = extract.get_ytplayer_config(stream_dict)['args']
+    extract.apply_descrambler(config_args, "url_encoded_fmt_stream_map")
+    assert "s" in config_args["url_encoded_fmt_stream_map"][0].keys()

tests/test_helpers.py

@@ -1,4 +1,7 @@
 # -*- coding: utf-8 -*-
+import gzip
+import io
+import json
 import os
 from unittest import mock
 
@@ -7,9 +10,11 @@ import pytest
 from pytube import helpers
 from pytube.exceptions import RegexMatchError
 from pytube.helpers import cache
+from pytube.helpers import create_mock_html_json
 from pytube.helpers import deprecated
 from pytube.helpers import setup_logger
 from pytube.helpers import target_directory
+from pytube.helpers import uniqueify
 
 
 def test_regex_search_no_match():
@@ -90,3 +95,75 @@ def test_setup_logger(logging):
     logging.getLogger.assert_called_with("pytube")
     logger.addHandler.assert_called()
     logger.setLevel.assert_called_with(20)
+
+
+@mock.patch('builtins.open', new_callable=mock.mock_open)
+@mock.patch('pytube.request.urlopen')
+def test_create_mock_html_json(mock_url_open, mock_open):
+    video_id = '2lAe1cqCOXo'
+    gzip_html_filename = 'yt-video-%s-html.json.gz' % video_id
+
+    # Get the pytube directory in order to navigate to /tests/mocks
+    pytube_dir_path = os.path.abspath(
+        os.path.join(
+            os.path.dirname(__file__),
+            os.path.pardir
+        )
+    )
+    pytube_mocks_path = os.path.join(pytube_dir_path, 'tests', 'mocks')
+    gzip_html_filepath = os.path.join(pytube_mocks_path, gzip_html_filename)
+
+    # Mock the responses to YouTube
+    mock_url_open_object = mock.Mock()
+
+    # Order is:
+    # 1. watch_html -- must have js match
+    # 2. vid_info_raw
+    # 3. js
+    mock_url_open_object.read.side_effect = [
+        b'"jsUrl":"base.js"',
+        b'vid_info_raw',
+        b'js_result',
+    ]
+    mock_url_open.return_value = mock_url_open_object
+
+    # Generate a json with sample html json
+    result_data = create_mock_html_json(video_id)
+
+    # Assert that a write was only made once
+    mock_open.assert_called_once_with(gzip_html_filepath, 'wb')
+
+    # The result data should look like this:
+    gzip_file = io.BytesIO()
+    with gzip.GzipFile(
+        filename=gzip_html_filename,
+        fileobj=gzip_file,
+        mode='wb'
+    ) as f:
+        f.write(json.dumps(result_data).encode('utf-8'))
+    gzip_data = gzip_file.getvalue()
+
+    file_handle = mock_open.return_value.__enter__.return_value
+
+    # For some reason, write gets called multiple times, so we have to
+    #  concatenate all the write calls to get the full data before we compare
+    #  it to the BytesIO object value.
+    full_content = b''
+    for call in file_handle.write.call_args_list:
+        args, kwargs = call
+        full_content += b''.join(args)
+
+    # The file header includes time metadata, so *occasionally* a single
+    #  byte will be off at the very beginning. In theory, this difference
+    #  should only affect bytes 5-8 (or [4:8] because of zero-indexing),
+    #  but I've excluded the 10-byte metadata header altogether from the
+    #  check, just to be safe.
+    # Source: https://en.wikipedia.org/wiki/Gzip#File_format
+    assert gzip_data[10:] == full_content[10:]
+
+
+def test_uniqueify():
+    non_unique_list = [1, 2, 3, 3, 4, 5]
+    expected = [1, 2, 3, 4, 5]
+    result = uniqueify(non_unique_list)
+    assert result == expected

tests/test_query.py

@@ -6,9 +6,9 @@ import pytest
 @pytest.mark.parametrize(
     ("test_input", "expected"),
     [
-        ({"progressive": True}, [18]),
-        ({"resolution": "720p"}, [136, 247]),
-        ({"res": "720p"}, [136, 247]),
+        ({"progressive": True}, [18, 22]),
+        ({"resolution": "720p"}, [22, 136, 247]),
+        ({"res": "720p"}, [22, 136, 247]),
         ({"fps": 30, "resolution": "480p"}, [135, 244]),
         ({"mime_type": "audio/mp4"}, [140]),
         ({"type": "audio"}, [140, 249, 250, 251]),
@@ -115,7 +115,7 @@ def test_order_by_non_numerical_ascending(cipher_signature):
 
 def test_order_by_with_none_values(cipher_signature):
     abrs = [s.abr for s in cipher_signature.streams.order_by("abr").asc()]
-    assert abrs == ["50kbps", "70kbps", "96kbps", "128kbps", "160kbps"]
+    assert abrs == ["50kbps", "70kbps", "96kbps", "128kbps", "160kbps", "192kbps"]
 
 
 def test_get_by_itag(cipher_signature):
@@ -138,13 +138,13 @@ def test_get_lowest_resolution(cipher_signature):
 
 
 def test_get_highest_resolution(cipher_signature):
-    assert cipher_signature.streams.get_highest_resolution().itag == 18
+    assert cipher_signature.streams.get_highest_resolution().itag == 22
 
 
 def test_filter_is_dash(cipher_signature):
     streams = cipher_signature.streams.filter(is_dash=False)
     itags = [s.itag for s in streams]
-    assert itags == [18, 399, 398, 397, 396, 395, 394]
+    assert itags == [18, 22, 399, 398, 397, 396, 395, 394]
 
 
 def test_get_audio_only(cipher_signature):
@@ -156,13 +156,13 @@ def test_get_audio_only_with_subtype(cipher_signature):
 
 
 def test_sequence(cipher_signature):
-    assert len(cipher_signature.streams) == 23
+    assert len(cipher_signature.streams) == 24
     assert cipher_signature.streams[0] is not None
 
 
 def test_otf(cipher_signature):
     non_otf = cipher_signature.streams.otf()
-    assert len(non_otf) == 23
+    assert len(non_otf) == 24
 
     otf = cipher_signature.streams.otf(True)
     assert len(otf) == 0

tests/test_streams.py

@@ -40,30 +40,25 @@ def test_filesize(cipher_signature):
 def test_filesize_approx(cipher_signature):
     stream = cipher_signature.streams[0]
 
-    assert stream.filesize_approx == 22350604
+    assert stream.filesize_approx == 28309811
     stream.bitrate = None
     assert stream.filesize_approx == 6796391
 
 
 def test_default_filename(cipher_signature):
-    expected = "PSY - GANGNAM STYLE(강남스타일) MV.mp4"
+    expected = "YouTube Rewind 2019 For the Record  YouTubeRewind.mp4"
     stream = cipher_signature.streams[0]
     assert stream.default_filename == expected
 
 
 def test_title(cipher_signature):
     expected = "title"
-    cipher_signature.player_config_args["title"] = expected
-    assert cipher_signature.title == expected
-
-    expected = "title2"
-    del cipher_signature.player_config_args["title"]
     cipher_signature.player_response = {"videoDetails": {"title": expected}}
     assert cipher_signature.title == expected
 
 
 def test_expiration(cipher_signature):
-    assert cipher_signature.streams[0].expiration == datetime(2020, 10, 24, 11, 7, 41)
+    assert cipher_signature.streams[0].expiration == datetime(2020, 10, 30, 5, 39, 41)
 
 
 def test_caption_tracks(presigned_video):
@@ -76,34 +71,42 @@ def test_captions(presigned_video):
 
 def test_description(cipher_signature):
     expected = (
-        "PSY - ‘I LUV IT’ M/V @ https://youtu.be/Xvjnoagk6GU\n"
-        "PSY - ‘New Face’ M/V @https://youtu.be/OwJPPaEyqhI\n\n"
-        "PSY - 8TH ALBUM '4X2=8' on iTunes @\n"
-        "https://smarturl.it/PSY_8thAlbum\n\n"
-        "PSY - GANGNAM STYLE(강남스타일) on iTunes @ "
-        "http://smarturl.it/PsyGangnam\n\n"
-        "#PSY #싸이 #GANGNAMSTYLE #강남스타일\n\n"
-        "More about PSY@\nhttp://www.youtube.com/officialpsy\n"
-        "http://www.facebook.com/officialpsy\n"
-        "http://twitter.com/psy_oppa\n"
-        "https://www.instagram.com/42psy42\n"
-        "http://iTunes.com/PSY\n"
-        "http://sptfy.com/PSY\n"
-        "http://weibo.com/psyoppa"
+        "In 2018, we made something you didn’t like. "
+        "For Rewind 2019, let’s see what you DID like.\n\n"
+        "Celebrating the creators, music and moments "
+        "that mattered most to you in 2019. \n\n"
+        "To learn how the top lists in Rewind were generated: "
+        "https://rewind.youtube/about\n\n"
+        "Top lists featured the following channels:\n\n"
+        "@1MILLION Dance Studio \n@A4 \n@Anaysa \n"
+        "@Andymation \n@Ariana Grande \n@Awez Darbar \n"
+        "@AzzyLand \n@Billie Eilish \n@Black Gryph0n \n"
+        "@BLACKPINK \n@ChapkisDanceUSA \n@Daddy Yankee \n"
+        "@David Dobrik \n@Dude Perfect \n@Felipe Neto \n"
+        "@Fischer's-フィッシャーズ- \n@Galen Hooks \n@ibighit \n"
+        "@James Charles \n@jeffreestar \n@Jelly \n@Kylie Jenner \n"
+        "@LazarBeam \n@Lil Dicky \n@Lil Nas X \n@LOUD \n@LOUD Babi \n"
+        "@LOUD Coringa \n@Magnet World \n@MrBeast \n"
+        "@Nilson Izaias Papinho Oficial \n@Noah Schnapp\n"
+        "@백종원의 요리비책 Paik's Cuisine \n@Pencilmation \n@PewDiePie \n"
+        "@SethEverman \n@shane \n@Shawn Mendes \n@Team Naach \n"
+        "@whinderssonnunes \n@워크맨-Workman \n@하루한끼 one meal a day \n\n"
+        "To see the full list of featured channels in Rewind 2019, "
+        "visit: https://rewind.youtube/about"
     )
     assert cipher_signature.description == expected
 
 
 def test_rating(cipher_signature):
-    assert cipher_signature.rating == 4.5375643
+    assert cipher_signature.rating == 2.0860765
 
 
 def test_length(cipher_signature):
-    assert cipher_signature.length == 252
+    assert cipher_signature.length == 337
 
 
 def test_views(cipher_signature):
-    assert cipher_signature.views == 3830838693
+    assert cipher_signature.views >= 108531745
 
 
 @mock.patch(
@@ -133,7 +136,7 @@ def test_download_with_prefix(cipher_signature):
         file_path = stream.download(filename_prefix="prefix")
         assert file_path == os.path.join(
             "/target",
-            "prefixPSY - GANGNAM STYLE(강남스타일) MV.mp4"
+            "prefixYouTube Rewind 2019 For the Record  YouTubeRewind.mp4"
         )
 
 
@@ -171,7 +174,7 @@ def test_download_with_existing(cipher_signature):
         file_path = stream.download()
         assert file_path == os.path.join(
             "/target",
-            "PSY - GANGNAM STYLE(강남스타일) MV.mp4"
+            "YouTube Rewind 2019 For the Record  YouTubeRewind.mp4"
         )
         assert not request.stream.called
 
@@ -192,7 +195,7 @@ def test_download_with_existing_no_skip(cipher_signature):
         file_path = stream.download(skip_existing=False)
         assert file_path == os.path.join(
             "/target",
-            "PSY - GANGNAM STYLE(강남스타일) MV.mp4"
+            "YouTube Rewind 2019 For the Record  YouTubeRewind.mp4"
         )
         assert request.stream.called
 
@@ -264,7 +267,7 @@ def test_thumbnail_when_in_details(cipher_signature):
 
 
 def test_thumbnail_when_not_in_details(cipher_signature):
-    expected = "https://img.youtube.com/vi/9bZkp7q19f0/maxresdefault.jpg"
+    expected = "https://img.youtube.com/vi/2lAe1cqCOXo/maxresdefault.jpg"
     cipher_signature.player_response = {}
     assert cipher_signature.thumbnail_url == expected