Spaces:
Running
Running
:recycle: [Refactor] HTMLFetcher: replace save_path with output_path
Browse files- networks/html_fetcher.py +9 -6
networks/html_fetcher.py
CHANGED
@@ -21,24 +21,27 @@ class HTMLFetcher:
|
|
21 |
)
|
22 |
|
23 |
def save_response(self):
|
24 |
-
self.
|
25 |
-
if not self.
|
26 |
-
self.
|
27 |
|
28 |
-
logger.success(f"Saving to: [{self.
|
29 |
|
30 |
-
with open(self.
|
31 |
wf.write(self.request_response.content)
|
32 |
|
33 |
def fetch(self, url):
|
34 |
self.url = url
|
35 |
self.send_request()
|
36 |
self.save_response()
|
|
|
37 |
|
38 |
|
39 |
if __name__ == "__main__":
|
40 |
url = (
|
41 |
-
"https://stackoverflow.com/questions/295135/turn-a-string-into-a-valid-filename"
|
|
|
|
|
42 |
)
|
43 |
fetcher = HTMLFetcher()
|
44 |
fetcher.fetch(url)
|
|
|
21 |
)
|
22 |
|
23 |
def save_response(self):
|
24 |
+
self.output_path = UrlToFilepathConverter().convert(self.url)
|
25 |
+
if not self.output_path.exists():
|
26 |
+
self.output_path.parent.mkdir(parents=True, exist_ok=True)
|
27 |
|
28 |
+
logger.success(f"Saving to: [{self.output_path}]")
|
29 |
|
30 |
+
with open(self.output_path, "wb") as wf:
|
31 |
wf.write(self.request_response.content)
|
32 |
|
33 |
def fetch(self, url):
|
34 |
self.url = url
|
35 |
self.send_request()
|
36 |
self.save_response()
|
37 |
+
return self.output_path
|
38 |
|
39 |
|
40 |
if __name__ == "__main__":
|
41 |
url = (
|
42 |
+
# "https://stackoverflow.com/questions/295135/turn-a-string-into-a-valid-filename"
|
43 |
+
# "https://www.liaoxuefeng.com/wiki/1016959663602400/1017495723838528"
|
44 |
+
"https://docs.python.org/zh-cn/3/tutorial/interpreter.html"
|
45 |
)
|
46 |
fetcher = HTMLFetcher()
|
47 |
fetcher.fetch(url)
|