Merge pull request #40 from hbmartin/reformatting

.flake8

@@ -1,3 +1,5 @@
 [flake8]
-ignore = E231,E203,W503
-max-line-length = 89
+ignore = E231,E203,W503,Q000,WPS111,WPS305,WPS348,WPS602,D400,DAR201,S101,DAR101,C812,D104,I001,WPS306,WPS214,D401,WPS229,WPS420,WPS230,WPS414,WPS114,WPS226,WPS442,C819,WPS601,T001,RST304,WPS410,WPS428,A003,A002,I003,WPS221,WPS326,WPS201,S405,DAR301,WPS210,WPS202,WPS213,WPS301,P103
+max-line-length = 89
+
+[isort]

Makefile

@@ -9,7 +9,8 @@ pipenv:
 	pipenv install --dev
 
 test:
-	pipenv run flake8
+	pipenv run flake8 pytube/
+	pipenv run flake8 tests/
 	pipenv run black pytube --check
 	pipenv run black tests --check
 	pipenv run mypy pytube

Pipfile

@@ -17,3 +17,10 @@ sphinx_rtd_theme = "*"
 mypy = "*"
 black = "==19.10b0"
 codecov = "*"
+flake8-bugbear ="*"
+flake8-comprehensions ="*"
+flake8-eradicate = "*"
+flake8-broken-line = "*"
+pep8-naming = "*"
+flake8-string-format = "*"
+flake8-quotes = "*"

pytube/__main__.py

@@ -67,12 +67,12 @@ class YouTube:
         self.watch_html: Optional[str] = None  # the html of /watch?v=<video_id>
         self.embed_html: Optional[str] = None
         self.player_config_args: Dict = {}  # inline js in the html containing
+        self.player_response: Dict = {}
         # streams
         self.age_restricted: Optional[bool] = None
         self.vid_descr: Optional[str] = None
 
         self.fmt_streams: List[Stream] = []
-        self.caption_tracks: List[Caption] = []
 
         # video_id part of /watch?v=<video_id>
         self.video_id = extract.video_id(url)
@@ -153,11 +153,9 @@ class YouTube:
             self.initialize_stream_objects(fmt)
 
         # load the player_response object (contains subtitle information)
-        self.player_config_args["player_response"] = json.loads(
-            self.player_config_args["player_response"]
-        )
+        self.player_response = json.loads(self.player_config_args["player_response"])
+        del self.player_config_args["player_response"]
 
-        self.initialize_caption_objects()
         logger.info("init finished successfully")
 
     def prefetch(self) -> None:
@@ -173,8 +171,7 @@ class YouTube:
         self.watch_html = request.get(url=self.watch_url)
         if (
             self.watch_html is None
-            or '<img class="icon meh" src="/yts/img'  # noqa: W503
-            not in self.watch_html  # noqa: W503
+            or '<img class="icon meh" src="/yts/img' not in self.watch_html
         ):
             raise VideoUnavailable(video_id=self.video_id)
 
@@ -214,26 +211,18 @@ class YouTube:
             )
             self.fmt_streams.append(video)
 
-    def initialize_caption_objects(self) -> None:
-        """Populate instances of :class:`Caption <Caption>`.
-
-        Take the unscrambled player response data, and use it to initialize
-        instances of :class:`Caption <Caption>`.
-
-        :rtype: None
+    @property
+    def caption_tracks(self) -> List[Caption]:
+        """Get a list of :class:`Caption <Caption>`.
 
+        :rtype: List[Caption]
         """
-        if "captions" not in self.player_config_args["player_response"]:
-            return
-        # https://github.com/nficano/pytube/issues/167
-        caption_tracks = (
-            self.player_config_args.get("player_response", {})
-            .get("captions", {})
+        raw_tracks = (
+            self.player_response.get("captions", {})
             .get("playerCaptionsTracklistRenderer", {})
             .get("captionTracks", [])
         )
-        for caption_track in caption_tracks:
-            self.caption_tracks.append(Caption(caption_track))
+        return [Caption(track) for track in raw_tracks]
 
     @property
     def captions(self) -> CaptionQuery:
@@ -258,9 +247,8 @@ class YouTube:
         :rtype: str
 
         """
-        player_response = self.player_config_args.get("player_response", {})
         thumbnail_details = (
-            player_response.get("videoDetails", {})
+            self.player_response.get("videoDetails", {})
             .get("thumbnail", {})
             .get("thumbnails")
         )
@@ -268,7 +256,7 @@ class YouTube:
             thumbnail_details = thumbnail_details[-1]  # last item has max size
             return thumbnail_details["url"]
 
-        return "https://img.youtube.com/vi/" + self.video_id + "/maxresdefault.jpg"
+        return f"https://img.youtube.com/vi/{self.video_id}/maxresdefault.jpg"
 
     @property
     def title(self) -> str:
@@ -278,9 +266,7 @@ class YouTube:
 
         """
         return self.player_config_args.get("title") or (
-            self.player_config_args.get("player_response", {})
-            .get("videoDetails", {})
-            .get("title")
+            self.player_response.get("videoDetails", {}).get("title")
         )
 
     @property
@@ -291,9 +277,7 @@ class YouTube:
 
         """
         return self.vid_descr or (
-            self.player_config_args.get("player_response", {})
-            .get("videoDetails", {})
-            .get("shortDescription")
+            self.player_response.get("videoDetails", {}).get("shortDescription")
         )
 
     @property
@@ -303,11 +287,7 @@ class YouTube:
         :rtype: float
 
         """
-        return (
-            self.player_config_args.get("player_response", {})
-            .get("videoDetails", {})
-            .get("averageRating")
-        )
+        return self.player_response.get("videoDetails", {}).get("averageRating")
 
     @property
     def length(self) -> int:
@@ -318,11 +298,7 @@ class YouTube:
         """
         return int(
             self.player_config_args.get("length_seconds")
-            or (
-                self.player_config_args.get("player_response", {})
-                .get("videoDetails", {})
-                .get("lengthSeconds")
-            )
+            or (self.player_response.get("videoDetails", {}).get("lengthSeconds"))
         )
 
     @property
@@ -332,22 +308,14 @@ class YouTube:
         :rtype: str
 
         """
-        return int(
-            self.player_config_args.get("player_response", {})
-            .get("videoDetails", {})
-            .get("viewCount")
-        )
+        return int(self.player_response.get("videoDetails", {}).get("viewCount"))
 
     @property
     def author(self) -> str:
         """Get the video author.
         :rtype: str
         """
-        return (
-            self.player_config_args.get("player_response", {})
-            .get("videoDetails", {})
-            .get("author", "unknown")
-        )
+        return self.player_response.get("videoDetails", {}).get("author", "unknown")
 
     def register_on_progress_callback(self, func: OnProgress):
         """Register a download progress callback function post initialization.

pytube/captions.py

@@ -4,10 +4,8 @@ import os
 import time
 import xml.etree.ElementTree as ElementTree
 from typing import Dict, Optional
-
 from pytube import request
 from html import unescape
-
 from pytube.helpers import safe_filename, target_directory
 
 
@@ -105,7 +103,6 @@ class Caption:
         :type filename_prefix: str or None
 
         :rtype: str
-
         """
         if title.endswith(".srt") or title.endswith(".xml"):
             filename = ".".join(title.split(".")[:-1])

pytube/cipher.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+
 """
 This module contains all logic necessary to decipher the signature.
 
@@ -26,7 +27,6 @@ logger = create_logger()
 
 def get_initial_function_name(js: str) -> str:
     """Extract the name of the function responsible for computing the signature.
-
     :param str js:
         The contents of the base.js asset file.
     :rtype: str
@@ -52,10 +52,10 @@ def get_initial_function_name(js: str) -> str:
     logger.debug("finding initial function name")
     for pattern in function_patterns:
         regex = re.compile(pattern)
-        results = regex.search(js)
-        if results:
+        function_match = regex.search(js)
+        if function_match:
             logger.debug("finished regex search, matched: %s", pattern)
-            return results.group(1)
+            return function_match.group(1)
 
     raise RegexMatchError(caller="get_initial_function_name", pattern="multiple")
 
@@ -112,11 +112,11 @@ def get_transform_object(js: str, var: str) -> List[str]:
     pattern = r"var %s={(.*?)};" % re.escape(var)
     logger.debug("getting transform object")
     regex = re.compile(pattern, flags=re.DOTALL)
-    results = regex.search(js)
-    if not results:
+    transform_match = regex.search(js)
+    if not transform_match:
         raise RegexMatchError(caller="get_transform_object", pattern=pattern)
 
-    return results.group(1).replace("\n", " ").split(", ")
+    return transform_match.group(1).replace("\n", " ").split(", ")
 
 
 def get_transform_map(js: str, var: str) -> Dict:
@@ -245,10 +245,10 @@ def parse_function(js_func: str) -> Tuple[str, int]:
     logger.debug("parsing transform function")
     pattern = r"\w+\.(\w+)\(\w,(\d+)\)"
     regex = re.compile(pattern)
-    results = regex.search(js_func)
-    if not results:
+    parse_match = regex.search(js_func)
+    if not parse_match:
         raise RegexMatchError(caller="parse_function", pattern=pattern)
-    fn_name, fn_arg = results.groups()
+    fn_name, fn_arg = parse_match.groups()
     return fn_name, int(fn_arg)
 
 
@@ -269,7 +269,7 @@ def get_signature(js: str, ciphered_signature: str) -> str:
     transform_plan = get_transform_plan(js)
     var, _ = transform_plan[0].split(".")
     transform_map = get_transform_map(js, var)
-    signature = [s for s in ciphered_signature]
+    signature = list(ciphered_signature)
 
     for js_func in transform_plan:
         name, argument = parse_function(js_func)

pytube/contrib/playlist.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
-"""Module to download a complete playlist from a youtube channel"""
+
+"""Module to download a complete playlist from a youtube channel."""
 
 import json
 import logging
@@ -15,9 +16,7 @@ logger = logging.getLogger(__name__)
 
 
 class Playlist:
-    """Handles all the task of manipulating and downloading a whole YouTube
-    playlist
-    """
+    """Load a YouTube playlist with URL or ID"""
 
     def __init__(self, url: str, proxies: Optional[Dict[str, str]] = None):
         if proxies:
@@ -28,18 +27,16 @@ class Playlist:
         except IndexError:  # assume that url is just the id
             self.playlist_id = url
 
-        self.playlist_url: str = (
-            "https://www.youtube.com/playlist?list=" + self.playlist_id
-        )
+        self.playlist_url = f"https://www.youtube.com/playlist?list={self.playlist_id}"
         self.html = request.get(self.playlist_url)
 
         # Needs testing with non-English
         self.last_update: Optional[date] = None
-        results = re.search(
+        date_match = re.search(
             r"<li>Last updated on (\w{3}) (\d{1,2}), (\d{4})</li>", self.html
         )
-        if results:
-            month, day, year = results.groups()
+        if date_match:
+            month, day, year = date_match.groups()
             self.last_update = datetime.strptime(
                 f"{month} {day:0>2} {year}", "%b %d %Y"
             ).date()
@@ -48,25 +45,26 @@ class Playlist:
 
     @staticmethod
     def _find_load_more_url(req: str) -> Optional[str]:
-        """Given an html page or a fragment thereof, looks for
-        and returns the "load more" url if found.
-        """
+        """Given an html page or fragment, returns the "load more" url if found."""
         match = re.search(
             r"data-uix-load-more-href=\"(/browse_ajax\?" 'action_continuation=.*?)"',
             req,
         )
         if match:
-            return "https://www.youtube.com" + match.group(1)
+            return f"https://www.youtube.com{match.group(1)}"
 
         return None
 
     @deprecated("This function will be removed in the future, please use .video_urls")
     def parse_links(self) -> List[str]:  # pragma: no cover
+        """ Deprecated function for returning list of URLs
+
+        :return: List[str]
+        """
         return self.video_urls
 
     def _paginate(self, until_watch_id: Optional[str] = None) -> Iterable[List[str]]:
-        """Parse the video links from the page source, extracts and
-        returns the /watch?v= part from video link href
+        """Parse the video links from the page source, yields the /watch?v= part from video link
         """
         req = self.html
         videos_urls = self._extract_videos(req)
@@ -113,6 +111,7 @@ class Playlist:
 
     def trimmed(self, video_id: str) -> Iterable[str]:
         """Retrieve a list of YouTube video URLs trimmed at the given video ID
+
         i.e. if the playlist has video IDs 1,2,3,4 calling trimmed(3) returns [1,2]
         :type video_id: str
             video ID to trim the returned list of playlist URLs at
@@ -121,16 +120,15 @@ class Playlist:
             List of video URLs from the playlist trimmed at the given ID
         """
         for page in self._paginate(until_watch_id=video_id):
-            for watch_path in page:
-                yield self._video_url(watch_path)
+            yield from (self._video_url(watch_path) for watch_path in page)
 
     @property  # type: ignore
     @cache
     def video_urls(self) -> List[str]:
         """Complete links of all the videos in playlist
+
         :rtype: List[str]
-        :returns:
-            List of video URLs
+        :returns: List of video URLs
         """
         return [
             self._video_url(video) for page in list(self._paginate()) for video in page
@@ -138,29 +136,27 @@ class Playlist:
 
     @property
     def videos(self) -> Iterable[YouTube]:
-        """Iterable of YouTube objects representing videos in this playlist
-        :rtype: Iterable[YouTube]
+        """Yields YouTube objects of videos in this playlist
+
+        :Yields: YouTube
         """
-        for url in self.video_urls:
-            yield YouTube(url)
+        yield from (YouTube(url) for url in self.video_urls)
 
     @deprecated(
         "This call is unnecessary, you can directly access .video_urls or .videos"
     )
     def populate_video_urls(self) -> List[str]:
         """Complete links of all the videos in playlist
+
         :rtype: List[str]
-        :returns:
-            List of video URLs
+        :returns: List of video URLs
         """
-
         return self.video_urls
 
     @deprecated("This function will be removed in the future.")
     def _path_num_prefix_generator(self, reverse=False):  # pragma: no cover
-        """
-        This generator function generates number prefixes, for the items
-        in the playlist.
+        """Generate number prefixes for the items in the playlist.
+
         If the number of digits required to name a file,is less than is
         required to name the last file,it prepends 0s.
         So if you have a playlist of 100 videos it will number them like:
@@ -185,9 +181,7 @@ class Playlist:
         reverse_numbering: bool = False,
         resolution: str = "720p",
     ) -> None:  # pragma: no cover
-        """Download all the videos in the the playlist. Initially, download
-        resolution is 720p (or highest available), later more option
-        should be added to download resolution of choice
+        """Download all the videos in the the playlist.
 
         :param download_path:
             (optional) Output path for the playlist If one is not
@@ -206,7 +200,6 @@ class Playlist:
             Video resolution i.e. "720p", "480p", "360p", "240p", "144p"
         :type resolution: str
         """
-
         logger.debug("total videos found: %d", len(self.video_urls))
         logger.debug("starting download")
 
@@ -231,22 +224,18 @@ class Playlist:
 
     @cache
     def title(self) -> Optional[str]:
-        """return playlist title (name)"""
-        open_tag = "<title>"
-        end_tag = "</title>"
-        pattern = re.compile(open_tag + "(.+?)" + end_tag)
+        """Extract playlist title
+
+        :return: playlist title (name)
+        :rtype: Optional[str]
+        """
+        pattern = re.compile("<title>(.+?)</title>")
         match = pattern.search(self.html)
 
         if match is None:
             return None
 
-        return (
-            match.group()
-            .replace(open_tag, "")
-            .replace(end_tag, "")
-            .replace("- YouTube", "")
-            .strip()
-        )
+        return match.group(1).replace("- YouTube", "").strip()
 
     @staticmethod
     def _video_url(watch_path: str):

pytube/exceptions.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+
 """Library specific exception definitions."""
 from typing import Union, Pattern
 

pytube/helpers.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+
 """Various helper functions implemented by pytube."""
 import functools
 import logging

pytube/monostate.py

@@ -1,9 +1,9 @@
+# -*- coding: utf-8 -*-
+
 import io
 from typing import Any, Optional
 from typing_extensions import Protocol
 
-# from __future__ import annotations
-
 
 class OnProgress(Protocol):
     def __call__(

pytube/query.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+
 """This module provides a query interface for media streams and captions."""
 from typing import List, Optional
 
@@ -232,6 +233,7 @@ class StreamQuery:
 
     def get_by_resolution(self, resolution: str) -> Optional[Stream]:
         """Get the corresponding :class:`Stream <Stream>` for a given resolution.
+
         Stream must be a progressive mp4.
 
         :param str resolution:

pytube/request.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+
 """Implements a simple wrapper around urlopen."""
 from typing import Any, Iterable, Dict
 from urllib.request import Request
@@ -25,10 +26,8 @@ def get(url) -> str:
 
 def stream(url: str, chunk_size: int = 8192) -> Iterable[bytes]:
     """Read the response in chunks.
-    :param str url:
-        The URL to perform the GET request for.
-    :param int chunk_size:
-        The size in bytes of each chunk. Defaults to 8*1024
+    :param str url: The URL to perform the GET request for.
+    :param int chunk_size: The size in bytes of each chunk. Defaults to 8*1024
     :rtype: Iterable[bytes]
     """
     response = _execute_request(url)

pytube/streams.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+
 """
 This module contains a container for stream manifest data.
 
@@ -169,7 +170,6 @@ class Stream:
         :returns:
             An os file system compatible filename.
         """
-
         filename = safe_filename(self.title)
         return f"{filename}.{self.subtype}"
 

pytube/version.py

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 __version__ = "9.6.1"
 
 if __name__ == "__main__":

tests/test_streams.py

@@ -199,29 +199,25 @@ def test_on_complete_hook(cipher_signature, mocker):
 
 def test_author(cipher_signature):
     expected = "Test author"
-    cipher_signature.player_config_args = {
-        "player_response": {"videoDetails": {"author": expected}}
-    }
+    cipher_signature.player_response = {"videoDetails": {"author": expected}}
     assert cipher_signature.author == expected
 
     expected = "unknown"
-    cipher_signature.player_config_args = {}
+    cipher_signature.player_response = {}
     assert cipher_signature.author == expected
 
 
 def test_thumbnail_when_in_details(cipher_signature):
     expected = "some url"
-    cipher_signature.player_config_args = {
-        "player_response": {
-            "videoDetails": {"thumbnail": {"thumbnails": [{"url": expected}]}}
-        }
+    cipher_signature.player_response = {
+        "videoDetails": {"thumbnail": {"thumbnails": [{"url": expected}]}}
     }
     assert cipher_signature.thumbnail_url == expected
 
 
 def test_thumbnail_when_not_in_details(cipher_signature):
     expected = "https://img.youtube.com/vi/9bZkp7q19f0/maxresdefault.jpg"
-    cipher_signature.player_config_args = {}
+    cipher_signature.player_response = {}
     assert cipher_signature.thumbnail_url == expected