Spaces:
Sleeping
Sleeping
Commit
·
6fa035b
1
Parent(s):
90af011
Add code
Browse files- app.py +44 -0
- processing.py +77 -0
- requirements.txt +1 -0
app.py
ADDED
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from processing import process_spaces
|
2 |
+
|
3 |
+
from apscheduler.schedulers.background import BackgroundScheduler
|
4 |
+
from huggingface_hub import hf_hub_url
|
5 |
+
from fastapi import FastAPI
|
6 |
+
from fastapi.responses import HTMLResponse
|
7 |
+
import uvicorn
|
8 |
+
import requests
|
9 |
+
|
10 |
+
def batch(iterable, n=1):
|
11 |
+
l = len(iterable)
|
12 |
+
for ndx in range(0, l, n):
|
13 |
+
yield iterable[ndx:min(ndx + n, l)]
|
14 |
+
|
15 |
+
|
16 |
+
def embed_theme_spaces():
|
17 |
+
|
18 |
+
subdomains = requests.get(hf_hub_url("freddyaboulton/gradio-theme-subdomains",
|
19 |
+
filename="subdomains.json",
|
20 |
+
repo_type="dataset")).json()
|
21 |
+
html = """"""
|
22 |
+
for row in batch(list(subdomains.keys()), n=4):
|
23 |
+
html += ("\n".join(["<div>"] +
|
24 |
+
[f"\t<iframe src='{subdomains[s]}' height='300', width='300'></iframe>" for s in row] +
|
25 |
+
["</div>"])
|
26 |
+
)
|
27 |
+
|
28 |
+
return html
|
29 |
+
|
30 |
+
|
31 |
+
|
32 |
+
app = FastAPI()
|
33 |
+
|
34 |
+
@app.get("/", response_class=HTMLResponse)
|
35 |
+
def index():
|
36 |
+
return embed_theme_spaces()
|
37 |
+
|
38 |
+
scheduler = BackgroundScheduler()
|
39 |
+
scheduler.add_job(func=process_spaces, trigger="interval", seconds=360)
|
40 |
+
scheduler.start()
|
41 |
+
|
42 |
+
if __name__ == "__main__":
|
43 |
+
uvicorn.run(app, port=7860)
|
44 |
+
|
processing.py
ADDED
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from __future__ import annotations
|
2 |
+
|
3 |
+
from typing import List, Dict, TypedDict
|
4 |
+
|
5 |
+
import huggingface_hub
|
6 |
+
from huggingface_hub.hf_api import SpaceInfo
|
7 |
+
import requests
|
8 |
+
from concurrent.futures import ThreadPoolExecutor
|
9 |
+
import os
|
10 |
+
import time
|
11 |
+
import random
|
12 |
+
import json
|
13 |
+
import datetime
|
14 |
+
|
15 |
+
class SpaceSubdomain(TypedDict):
|
16 |
+
name: str
|
17 |
+
subdomain: str
|
18 |
+
|
19 |
+
|
20 |
+
repo = huggingface_hub.Repository(
|
21 |
+
local_dir="data",
|
22 |
+
repo_type="dataset",
|
23 |
+
clone_from="freddyaboulton/gradio-theme-subdomains",
|
24 |
+
token=os.getenv("HF_TOKEN")
|
25 |
+
)
|
26 |
+
repo.git_pull()
|
27 |
+
|
28 |
+
|
29 |
+
api = huggingface_hub.HfApi(token=os.getenv("HF_TOKEN"))
|
30 |
+
|
31 |
+
def get_theme_preview_spaces() -> List[SpaceInfo]:
|
32 |
+
return api.list_spaces(filter="gradio-theme")
|
33 |
+
|
34 |
+
def get_subdomain(space_name: SpaceInfo) -> SpaceSubdomain | None:
|
35 |
+
time.sleep(random.random())
|
36 |
+
print(f"processing {space_name.id}")
|
37 |
+
if not space_name.id:
|
38 |
+
print(f"no space_name for {space_name}")
|
39 |
+
return None
|
40 |
+
subdomain: str | None = (
|
41 |
+
requests.get(
|
42 |
+
f"https://huggingface.co/api/spaces/{space_name.id}/host",
|
43 |
+
headers={"Authorization": f"Bearer {os.getenv('HF_TOKEN')}"}
|
44 |
+
)
|
45 |
+
.json()
|
46 |
+
.get("host")
|
47 |
+
)
|
48 |
+
if not subdomain:
|
49 |
+
print(f"no subdomain for {space_name.id}")
|
50 |
+
return None
|
51 |
+
return {'name': space_name.id, 'subdomain': subdomain}
|
52 |
+
|
53 |
+
|
54 |
+
def get_all_subdomains(spaces: List[SpaceInfo], subdomains: Dict[str, str]) -> Dict[str, str]:
|
55 |
+
|
56 |
+
not_parsed_yet = [space for space in spaces if space.id not in subdomains]
|
57 |
+
|
58 |
+
with ThreadPoolExecutor(max_workers=10) as executor:
|
59 |
+
new_subdomains = executor.map(get_subdomain, not_parsed_yet)
|
60 |
+
|
61 |
+
new_subdomains = {s["name"]: s["subdomain"] for s in new_subdomains if s}
|
62 |
+
|
63 |
+
new_subdomains.update(subdomains)
|
64 |
+
|
65 |
+
return new_subdomains
|
66 |
+
|
67 |
+
|
68 |
+
def process_spaces():
|
69 |
+
theme_spaces = get_theme_preview_spaces()
|
70 |
+
subdomains: Dict[str, str] = json.load(open("data/subdomains.json"))
|
71 |
+
|
72 |
+
all_subdomains = get_all_subdomains(theme_spaces, subdomains)
|
73 |
+
|
74 |
+
json.dump(all_subdomains, open("data/subdomains.json", "w"))
|
75 |
+
repo.push_to_hub(blocking=False, commit_message=f"Updating data at {datetime.datetime.now()}")
|
76 |
+
|
77 |
+
|
requirements.txt
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
apscheduler
|