Add infinite retries to fetching leaderboard data

app.py

@@ -306,24 +306,32 @@ def restart_space():
 
 
 def main():
-    subtensor, metagraph = get_subtensor_and_metagraph()
+    # To avoid leaderboard failures, infinitely try until we get all data
+    # needed to populate the dashboard
+    while True:
+        try:
+            subtensor, metagraph = get_subtensor_and_metagraph()
 
-    model_data: List[ModelData] = get_subnet_data(subtensor, metagraph)
-    model_data.sort(key=lambda x: x.incentive, reverse=True)
+            model_data: List[ModelData] = get_subnet_data(subtensor, metagraph)
+            model_data.sort(key=lambda x: x.incentive, reverse=True)
 
-    vali_runs = get_wandb_runs(project=VALIDATOR_WANDB_PROJECT, filters={"config.type": "validator", "config.uid": 238})
+            vali_runs = get_wandb_runs(project=VALIDATOR_WANDB_PROJECT, filters={"config.type": "validator", "config.uid": 238})
 
-    scores = get_scores([x.uid for x in model_data], vali_runs)
+            scores = get_scores([x.uid for x in model_data], vali_runs)
 
-    current_block = metagraph.block.item()
-    next_epoch_block = next_epoch(subtensor, current_block)
+            current_block = metagraph.block.item()
+            next_epoch_block = next_epoch(subtensor, current_block)
 
-    validator_df = get_validator_weights(metagraph)
-    weight_keys = set()
-    for uid, stats in validator_df.items():
-        weight_keys.update(stats[-1].keys())
-        
-    benchmarks, benchmark_timestamp = get_benchmarks()
+            validator_df = get_validator_weights(metagraph)
+            weight_keys = set()
+            for uid, stats in validator_df.items():
+                weight_keys.update(stats[-1].keys())
+                
+            benchmarks, benchmark_timestamp = get_benchmarks()
+            break
+        except Exception as e:
+            print(f"Failed to get data: {e}")
+            time.sleep(30)            
 
     demo = gr.Blocks(css=".typewriter {font-family: 'JMH Typewriter', sans-serif;}")
     with demo: