updates

.env.example

@@ -0,0 +1,2 @@
+USER=
+PERSONAL_ACCESS_TOKEN=

.gitignore

@@ -1,2 +1,5 @@
 .DS_Store
-.coverage
+.coverage
+*.txt
+*.html
+.env

Pipfile

@@ -6,6 +6,9 @@ name = "pypi"
 [packages]
 streamlit = "*"
 requests = "*"
+networkx = "*"
+pyvis = "*"
+orjson = "*"
 
 [dev-packages]
 black = "*"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "5f6e3997d5cf785edec4ab60f34ebef181b10ce1362ec421a4f64eb1562132bf"
+            "sha256": "0006fdd10642d6c40ca9dc5aaaa8f3d586465b1f5c1a306b70eb71b3559d707f"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -375,6 +375,14 @@
             "markers": "python_version >= '3.7'",
             "version": "==3.1.2"
         },
+        "jsonpickle": {
+            "hashes": [
+                "sha256:7b272918b0554182e53dc340ddd62d9b7f902fec7e7b05620c04f3ccef479a0e",
+                "sha256:de7f2613818aa4f234138ca11243d6359ff83ae528b2185efdd474f62bcf9ae1"
+            ],
+            "markers": "python_version >= '2.7'",
+            "version": "==2.2.0"
+        },
         "jsonschema": {
             "hashes": [
                 "sha256:1c92d2db1900b668201f1797887d66453ab1fbfea51df8e4b46236689c427baf",
@@ -508,6 +516,14 @@
             "markers": "python_version >= '3.5'",
             "version": "==1.5.5"
         },
+        "networkx": {
+            "hashes": [
+                "sha256:51d6ae63c24dcd33901357688a2ad20d6bcd38f9a4c5307720048d3a8081059c",
+                "sha256:ae99c9b0d35e5b4a62cf1cfea01e5b3633d8d02f4a0ead69685b6e7de5b85eab"
+            ],
+            "index": "pypi",
+            "version": "==2.8.2"
+        },
         "notebook": {
             "hashes": [
                 "sha256:709b1856a564fe53054796c80e17a67262071c86bfbdfa6b96aaa346113c555a",
@@ -544,6 +560,44 @@
             "markers": "python_version < '3.10' and platform_machine != 'aarch64' and platform_machine != 'arm64'",
             "version": "==1.22.4"
         },
+        "orjson": {
+            "hashes": [
+                "sha256:033dd5f91a8a967a007d3d05cbabec67040d6ff3e159ea17d3f681c3114a0d78",
+                "sha256:26c64da280c9e097081d12047f13b4adba776b19885da5e488c093d4a1461056",
+                "sha256:31367b5d8389373aff1742b66608c4bff318a9015d94981a8c1919e82fe983ca",
+                "sha256:32f384ff9dd555ee21508887b12316d8bd04921b396c876b4d4c87a30a3c8a13",
+                "sha256:349514d69ce089b0e39014345907318ddba8ceca32a187635601c391c36ecdd6",
+                "sha256:37b41c8869347388b1794a1c92c5e981ab764638f62e026252026a650b1b266c",
+                "sha256:3a1e2dfa7ba8adb7511f3560e968ff2a66e7d0cd2f454219a0ab778c3c2e1a5c",
+                "sha256:3ae89fd45bc9c72dcc0a489aa2411f139ee8a32468c387188be21d25f20f83d3",
+                "sha256:4175929ca77338e6a57ff232c0e80443411ac0b489bfff755988ae70e3f62a97",
+                "sha256:42376b0330cbbe5864b480de16a48f4c82aae95dca9cdcf81490e7ca87cc131a",
+                "sha256:43b9a44b42c67adbc02fc86efacf27a374b09971cd58e0cd9739b8a748d19be5",
+                "sha256:454a4d8c81882cbff19eaed90d0a4e42602970c686c8cd34071c8097e3dfdb5c",
+                "sha256:4f657a16f81b0497e5c67b3c151d9eb8c99d2d3a7ab996500e9ae453e8b0e0fe",
+                "sha256:5730e44fc20891cadea7d163a2dad723f95cf81199e1a02dac339a11437a999d",
+                "sha256:6165914b1a209458201bf0a99dcf5f44f58477ba23ac71d7b5e4ca197e174f30",
+                "sha256:696661c6b6e58361aba0b14a2c5977c049f481bc7fe41759a55e86b13b361905",
+                "sha256:6fdef8939f528dd9386c4941e88227a3ccf124c8278ebc7e98533294ae446ef1",
+                "sha256:71c285488c5f767e102a389f9efb11e93e6345247d60043efeffa616a3056945",
+                "sha256:7b03a3f32cd5fcdd8460de690579d1dfa1965bd333b89bd3d202907c7b49ada4",
+                "sha256:7e309db07b13d84bc5eb6ffdfd46f00b2301ce78871809e177f599db3f31fe24",
+                "sha256:8d877467096dc117500a5ed38238085a81518252db3991793a8468ca97445e6c",
+                "sha256:906a33c2fe834cb47daafd09d77405260caa5fa1354219bec5df9ac2b4e909fa",
+                "sha256:a23292d6093748eee3f7ed85dbe6c9abd24d4d399d7044bc323ff39834966d6c",
+                "sha256:aa0919735afbdeada9687347fa7963ca9732c60b1005832ac9d4853a9cb48be9",
+                "sha256:ad0faec8ee89cd50a486804b7d9a97016a1d2074298ceadbc75efc6691030b36",
+                "sha256:b71915140261916e50dbf62dec1b448c159a789e23bc89b3ad1e6516f677036f",
+                "sha256:c4d9d1edd7c92d0b35082ae3a230705206d6beb3c07a8c72ebdd980820598f9c",
+                "sha256:d4a380c164f8c40660ce8f96791695aa74d32a2a45d4038a2a4826d9ec2c61f6",
+                "sha256:f65b7d87534c567136d73f9bcdff46ad5ded2aabe8e89be7ba97395129d48e72",
+                "sha256:f6e691fcf0e03575bef7efd24b331282ad3c1df75855be368e4c3c2cfba6967f",
+                "sha256:f7b628f4dcfc0b726ede7d2024cf79107d54851451e15edeae2f4ee55554f3c6",
+                "sha256:fc7ecc7b38ebf6ac072efa209c5b8d02eb7af393a8a2c812fc01dcb037a86c48"
+            ],
+            "index": "pypi",
+            "version": "==3.7.1"
+        },
         "packaging": {
             "hashes": [
                 "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb",
@@ -878,6 +932,13 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
             "version": "==0.1.0.post0"
         },
+        "pyvis": {
+            "hashes": [
+                "sha256:29b94d0a10a01ac790d262318680369baa9e5eb8524c355758f1c3799f0c3965"
+            ],
+            "index": "pypi",
+            "version": "==0.2.1"
+        },
         "pyzmq": {
             "hashes": [
                 "sha256:057176dd3f5ccf5aad4abd662d76b6a39bbf799baaf2f39cd4fdaf2eab326e43",
@@ -954,7 +1015,7 @@
                 "sha256:4c586de507202505346f3e32d1363eb9ed6932f0c2f63184dea88983ff4971e2",
                 "sha256:d2bbd99c320a2532ac71ff6a3164867884357da3e3301f0240090c5d2fdac7ec"
             ],
-            "markers": "python_version < '4' and python_full_version >= '3.6.3'",
+            "markers": "python_full_version >= '3.6.3' and python_full_version < '4.0.0'",
             "version": "==12.4.4"
         },
         "semver": {
@@ -1127,7 +1188,7 @@
                 "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14",
                 "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_full_version < '4.0.0'",
             "version": "==1.26.9"
         },
         "validators": {

README.md

@@ -5,16 +5,35 @@
 The main purpose of the app is to allow Python developers navigate Python code base much easier by showing dependencies
 among files included in the directory with better visualization.
 
+## Setup
+
+- Python version: 3.8
+- `pipenv install`
+- `cp .env.example .env`: fill in your GitHub username and personal access token if you need to increase [GitHub API requests rate limiting](https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting)
+- `make run`
+- Also refer to References to understand settings behind the project
+
 ## Thoughts on solving the problem
 
 1. Build a prototype without UI interface to show dependencies among files included in the directory(output) given user's
 input to one GitHub public repo's URL(input)
-2. Build a streamlit app to show results using some network visualization tools
+2. Build a `streamlit` app to show results using some network visualization tools
 
 ## Used technologies
 
-- Streamlit
+- `networkx`: network analysis
+- `pyvis`: network visualization
+- `streamlit`: web app
+
+## TODOs
 
+- [ ] Build a prototype
+  - [ ] Finish `generate_imports_graph` implementation
+  - [x] Fetch python files given public GitHub repo url(owner, repo, path, ref)
+  - [x] Use `ast` to parse imports among given python files
+  - [x] Generate a basic `networkx` graph given python imports
+  - [x] Visualize a basic `networkx` graph using `pyvis`
+- [ ] Build a `streamlit` app
 ## References
 
 - [How to set up a perfect Python project](https://sourcery.ai/blog/python-best-practices/)

app.py

@@ -1,6 +1,9 @@
-from py_code_analyzer.code_analyzer import CodeAnalyzer
-from py_code_analyzer.code_fetcher import get_repository_python_files
+from py_code_analyzer.code_fetcher import CodeFetcher
+from py_code_analyzer.code_imports_analyzer import CodeImportsAnalyzer
+from py_code_analyzer.imports_graph_visualizer import ImportsGraphVisualizer
 
-python_files = get_repository_python_files("cyyeh", "gradio")
-
-CodeAnalyzer(python_files).analyze_imports().report()
+python_files = CodeFetcher().get_python_files("cyyeh", "gradio", "gradio")
+code_imports_analyzer = (
+    CodeImportsAnalyzer(python_files).analyze_imports().generate_imports_graph()
+)
+ImportsGraphVisualizer().visualize(code_imports_analyzer.imports_graph)

py_code_analyzer/code_fetcher.py

@@ -1,19 +1,48 @@
-"""This file deals with every detail of how to get all python files in the given directory
+"""CodeFetcher deals with every detail of
+how to get all python files in the given directory
 """
+import os
+
 import requests
 
+# to increase api rate limiting
+# https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting
+USER = os.environ.get("PERSONAL_ACCESS_TOKEN", "")
+PERSONAL_ACCESS_TOKEN = os.environ.get("PERSONAL_ACCESS_TOKEN", "")
+
+
+class CodeFetcher:
+    def get_python_files(
+        self,
+        owner: str,
+        repo: str,
+        path: str = "",
+        ref: str = "",
+        recursive: bool = True,
+    ):
+        """https://docs.github.com/en/rest/repos/contents#get-repository-content"""
 
-def get_repository_python_files(owner: str, repo: str, path: str = "", ref: str = ""):
-    """https://docs.github.com/en/rest/repos/contents#get-repository-content"""
-    api_url = f"https://api.github.com/repos/{owner}/{repo}/contents/{path}"
-    if ref:
-        api_url += f"?ref={ref}"
+        api_url = (
+            f"https://api.github.com/repos/{owner}/{repo}/contents/{path}"
+            if not USER or not PERSONAL_ACCESS_TOKEN
+            else f"https://{USER}:{PERSONAL_ACCESS_TOKEN}@api.github.com/repos/{owner}/{repo}/contents/{path}"
+        )
+        if ref:
+            api_url += f"?ref={ref}"
 
-    python_files = []
-    api_results = requests.get(api_url).json()
+        python_files = []
+        api_results = requests.get(api_url).json()
 
-    for result in api_results:
-        if result["type"] == "file" and result["name"].endswith(".py"):
-            python_files.append(result)
+        for result in api_results:
+            if result["type"] == "file" and result["name"].endswith(".py"):
+                python_files.append(result)
+            elif (
+                recursive
+                and result["type"] == "dir"
+                and not result["name"].startswith(".")
+            ):
+                python_files += self.get_python_files(
+                    owner, repo, path=result["path"], recursive=recursive
+                )
 
-    return python_files
+        return python_files

py_code_analyzer/{code_analyzer.py → code_imports_analyzer.py}

@@ -1,11 +1,15 @@
+"""CodeImportsAnalyzer uses the ast module from Python's standard library
+to get what modules are imported in given python files, then uses networkx to generate imports graph
+"""
 import ast
 from pprint import pprint
 
+import networkx as nx
 import requests
 
 
-class CodeAnalyzer:
-    class NodeVisitor(ast.NodeVisitor):
+class CodeImportsAnalyzer:
+    class _NodeVisitor(ast.NodeVisitor):
         def __init__(self, imports):
             self.imports = imports
 
@@ -25,16 +29,35 @@ class CodeAnalyzer:
 
     def __init__(self, python_files):
         self.imports = []
+        self.imports_graph = nx.DiGraph()  # imports_graph is a directed graph
         self.python_files = python_files
-        self._node_visitor = CodeAnalyzer.NodeVisitor(self.imports)
+        self._node_visitor = CodeImportsAnalyzer._NodeVisitor(self.imports)
 
     def analyze_imports(self):
         for python_file in self.python_files:
             program = requests.get(python_file["download_url"]).text
             tree = ast.parse(program)
-            self.imports += [{"file_name": python_file["name"], "imports": []}]
+            self.imports += [
+                {
+                    "file_name": python_file["name"],
+                    "file_path": python_file["path"],
+                    "imports": [],
+                }
+            ]
             self._node_visitor.visit(tree)
         return self
 
+    def generate_imports_graph(self):
+        for _import in self.imports:
+            _nodes = _import["file_path"].split("/")
+            if len(_nodes):
+                if len(_nodes) > 1:
+                    for first_node, second_node in zip(_nodes, _nodes[1:]):
+                        self.imports_graph.add_edge(first_node, second_node)
+                else:
+                    self.imports_graph.add_node(_nodes[0])
+
+        return self
+
     def report(self):
         pprint(self.imports)

py_code_analyzer/imports_graph_visualizer.py

@@ -0,0 +1,16 @@
+import networkx as nx
+from pyvis.network import Network
+
+
+class ImportsGraphVisualizer:
+    @classmethod
+    def visualize(
+        cls,
+        imports_graph: nx.Graph,
+        width: int = 100,
+        height: int = 100,
+        display_html_name: str = "nx.html",
+    ):
+        _pyvis_network = Network(f"{width}%", f"{height}%")
+        _pyvis_network.from_nx(imports_graph)
+        _pyvis_network.show(display_html_name)