Spaces:
Sleeping
Sleeping
File size: 2,774 Bytes
71f183c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 |
import numpy as np
import sys
from modelguidedattacks import results
results_list = results.load_all_results()
filter = {
"loss": results.in_set(["cvxproj"]),
"model": results.eq("resnet50"),
"k": results.eq(20),
"binary_search_steps": results.eq(1),
"unguided_iterations": results.eq(60),
# "topk_loss_coef_upper": results.eq(20),
# "unguided_lr": results.eq(0.002),
"cvx_proj_margin": results.eq(0.2),
"topk_loss_coef_upper": results.gte(12)
# "seed": results.eq(10),
}
filtered_results = results.filter_results(filter, results_list)
combined_results = {}
for result in filtered_results:
for key in result:
if key not in combined_results:
combined_results[key] = []
combined_results[key].append(result[key])
unique_runs = len(np.unique(combined_results["seed"]))
print ("Stats from", len(filtered_results))
# assert len(combined_results["seed"]) == unique_runs
for key, val in list(combined_results.items()):
if key in ["ASR", "L1", "L2", "L_inf"]:
val = np.array(val)
combined_results[f"{key}_mean"] = np.mean(val[np.isfinite(val)])
combined_results[f"{key}_median"] = np.median(val[np.isfinite(val)])
# Coupled results
best_asr_idx = np.argmax(combined_results["ASR"])
best_asr = combined_results["ASR"][best_asr_idx]
best_l1 = combined_results["L1"][best_asr_idx]
best_l2 = combined_results["L2"][best_asr_idx]
best_linf = combined_results["L_inf"][best_asr_idx]
combined_results["ASR_best"] = best_asr
combined_results["L1_best"] = best_l1
combined_results["L2_best"] = best_l2
combined_results["L_inf_best"] = best_linf
worst_asr_idx = np.argmin(combined_results["ASR"])
worst_asr = combined_results["ASR"][worst_asr_idx]
worst_l1 = combined_results["L1"][worst_asr_idx]
worst_l2 = combined_results["L2"][worst_asr_idx]
worst_linf = combined_results["L_inf"][worst_asr_idx]
combined_results["ASR_worst"] = worst_asr
combined_results["L1_worst"] = worst_l1
combined_results["L2_worst"] = worst_l2
combined_results["L_inf_worst"] = worst_linf
draw_keys = ["best", "mean", "worst"]
val_keys = ["ASR", "L1", "L2", "L_inf"]
print ("---------------")
for draw_key in draw_keys:
for val_key in val_keys:
key = val_key + "_" + draw_key
val = combined_results[key]
print (key, val)
print ("---------------")
for draw_key in draw_keys:
for val_key in val_keys:
key = val_key + "_" + draw_key
val = combined_results[key]
if np.isinf(val):
val = "N/A"
sep = "&"
if isinstance(val, str):
sys.stdout.write(f"{val} {sep} ")
elif "inf" in key:
sys.stdout.write(f"{val:.3f} {sep} ")
else:
sys.stdout.write(f"{val:.2f} {sep} ") |