Duplicate from renumics/cifar10-outlier

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,2 @@
+
+.venv/

Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.9
+
+WORKDIR /code
+ENV HOME=/code
+
+RUN apt install curl
+RUN pip install pip -U
+
+RUN pip install renumics-spotlight==1.2.0rc2
+
+RUN pip install datasets
+COPY prepare.py .
+RUN python prepare.py
+
+COPY . .
+RUN mkdir -p /code/.cache
+RUN chmod -R 777 /code
+CMD ["python", "run.py"]

README.md

@@ -0,0 +1,20 @@
+---
+title: Explore Outliers in CIFAR-10 with Spotlight
+emoji: 📊
+colorFrom: gray
+colorTo: blue
+sdk: docker
+pinned: false
+license: mit
+app_file: run.py
+datasets:
+- renumics/cifar10-outlier
+- cifar10
+tags:
+- renumics
+- spotlight
+- EDA
+duplicated_from: renumics/cifar10-outlier
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

prepare.py

@@ -0,0 +1,42 @@
+import pickle
+import datasets
+import os
+import umap
+
+
+if __name__ == "__main__":
+    cache_file = "dataset_cache.pkl"
+    if os.path.exists(cache_file):
+        # Load dataset from cache
+        with open(cache_file, "rb") as file:
+            dataset = pickle.load(file)
+        print("Dataset loaded from cache.")
+    else:
+        # Load dataset using datasets.load_dataset()
+        ds = datasets.load_dataset("renumics/cifar10-outlier", split="train")
+        print("Dataset loaded using datasets.load_dataset().")
+
+        df = ds.rename_columns({"img": "image", "label": "labels"}).to_pandas()
+        df["label_str"] = df["labels"].apply(lambda x: ds.features["label"].int2str(x))
+
+        # df = df[:1000]
+
+        # precalculate umap embeddings
+        df["embedding_ft_precalc"] = umap.UMAP(
+            n_neighbors=70, min_dist=0.5, random_state=42
+        ).fit_transform(df["embedding_ft"].tolist()).tolist()
+        print("Umap for ft done")
+
+
+        df["embedding_foundation_precalc"] = umap.UMAP(
+            n_neighbors=70, min_dist=0.5, random_state=42
+        ).fit_transform(df["embedding_foundation"].tolist()).tolist()
+
+        print("Umap for base done")
+
+        # Save dataset to cache
+        with open(cache_file, "wb") as file:
+            pickle.dump(df, file)
+
+        print("Dataset saved to cache.")
+

requirements.txt

@@ -0,0 +1,2 @@
+renumics-spotlight
+datasets

run.py

@@ -0,0 +1,37 @@
+import pickle
+from renumics import spotlight
+import os
+
+if __name__ == "__main__":
+    cache_file = "dataset_cache.pkl"
+    if os.path.exists(cache_file):
+        # Load dataset from cache
+        with open(cache_file, "rb") as file:
+            df = pickle.load(file)
+        print("Dataset loaded from cache.")
+
+
+        while True:
+            dtypes = {
+                "nn_image": spotlight.Image,
+                "image": spotlight.Image,
+                "embedding_ft": spotlight.Embedding,
+                "embedding_foundation": spotlight.Embedding,
+                "embedding_ft_precalc": spotlight.Embedding,
+                "embedding_foundation_precalc": spotlight.Embedding,
+            }
+            view = spotlight.show(
+                df,
+                dtype=dtypes,
+                layout="https://spotlight.renumics.com/resources/layout_pre_post_ft_hf.json",
+                port=7860, 
+                host="0.0.0.0",
+                allow_filebrowsing=False 
+            )
+
+            view.close()
+
+    else:
+        print(f"Dataset {cache_file} not found. Please run prepare.py first.")
+
+

spotlight-layout.json

@@ -0,0 +1 @@
+{"orientation":"vertical","children":[{"kind":"split","weight":55.52608311229001,"orientation":"horizontal","children":[{"kind":"tab","weight":41.16700201207244,"children":[{"kind":"widget","name":"Table","type":"table","config":{"tableView":"full","visibleColumns":["coarse_label_str","fine_label_prediction_str","fine_label_str","image","__idx__"],"sorting":[["__idx__","DESC"]],"orderByRelevance":false,"columnWidths":{"coarse_label":92,"coarse_label_str":165,"embedding_reduced":128,"fine_label":92,"fine_label_prediction":92,"fine_label_prediction_error":92,"fine_label_prediction_str":244,"fine_label_str":194,"full_image":256,"image":200,"split":256,"__idx__":92,"__last_edited_at__":192,"__last_edited_by__":256}}}]},{"kind":"tab","weight":33.66487769877306,"children":[{"kind":"widget","name":"Similarity Map","type":"similaritymap","config":{"placeBy":null,"reductionMethod":null,"colorBy":"fine_label_str","sizeBy":null,"filter":false,"umapNNeighbors":20,"umapMetric":null,"umapMinDist":0.15,"pcaNormalization":null,"umapMenuLocalGlobalBalance":null,"umapMenuIsAdvanced":false}},{"kind":"widget","name":"Scatter Plot","type":"scatterplot","config":{"xAxisColumn":null,"yAxisColumn":null,"colorBy":null,"sizeBy":null,"filter":false}}]},{"kind":"tab","weight":25.168120289154498,"children":[{"kind":"widget","name":"Histogram","type":"histogram","config":{"columnKey":"fine_label_str","stackByColumnKey":"fine_label_prediction_error","filter":false}}]}]},{"kind":"tab","weight":44.47391688770999,"children":[{"kind":"widget","name":"Inspector","type":"inspector","config":{"views":[{"view":"ImageView","key":"es3NP2fbYeG5TzTQqCzZKL","name":"image","columns":["image"]},{"view":"TextLens","columns":["fine_label_str"],"name":"view","key":"7262001a-1270-467c-bfe8-6b6883adbc7d"},{"view":"TextLens","columns":["fine_label_prediction_str"],"name":"view","key":"f72a8ae9-17c4-4097-b454-79a6f60865db"},{"view":"ScalarView","columns":["fine_label_prediction_error"],"name":"view","key":"be4d6afe-9093-4581-b363-df11d3c6d34a"}],"visibleColumns":4}}]}]}