update path in crawlies
Browse filesSigned-off-by: Balazs Horvath <acsipont@gmail.com>
- crawl/crawl +2 -2
- crawl/crawl_wikipedia +2 -2
crawl/crawl
CHANGED
@@ -124,9 +124,9 @@ def save_result(target_url):
|
|
124 |
|
125 |
# Choose the appropriate base path based on the operating system
|
126 |
if platform.system() == "Windows":
|
127 |
-
base_path = "E:\\datasets\\
|
128 |
else:
|
129 |
-
base_path = "/home/kade/datasets/
|
130 |
|
131 |
save_dir = os.path.join(base_path, sanitized_title)
|
132 |
os.makedirs(save_dir, exist_ok=True)
|
|
|
124 |
|
125 |
# Choose the appropriate base path based on the operating system
|
126 |
if platform.system() == "Windows":
|
127 |
+
base_path = "E:\\datasets\\ragpile\\Saved Websites\\"
|
128 |
else:
|
129 |
+
base_path = "/home/kade/datasets/ragpile/Saved Websites"
|
130 |
|
131 |
save_dir = os.path.join(base_path, sanitized_title)
|
132 |
os.makedirs(save_dir, exist_ok=True)
|
crawl/crawl_wikipedia
CHANGED
@@ -126,9 +126,9 @@ def save_result(target_url):
|
|
126 |
|
127 |
# Choose the appropriate base path based on the operating system
|
128 |
if platform.system() == "Windows":
|
129 |
-
base_path = "E:\\
|
130 |
else:
|
131 |
-
base_path = "/home/kade/
|
132 |
|
133 |
save_dir = os.path.join(base_path, sanitized_title)
|
134 |
os.makedirs(save_dir, exist_ok=True)
|
|
|
126 |
|
127 |
# Choose the appropriate base path based on the operating system
|
128 |
if platform.system() == "Windows":
|
129 |
+
base_path = "E:\\ragpile\\Saved Websites\\"
|
130 |
else:
|
131 |
+
base_path = "/home/kade/datasets/ragpile/Saved Websites/"
|
132 |
|
133 |
save_dir = os.path.join(base_path, sanitized_title)
|
134 |
os.makedirs(save_dir, exist_ok=True)
|