Add similarity distribution analysis for debugging embedding quality
- Add --similarity-details flag to test_logo_detection.py - Track true positive, false positive, and missed detection similarities - Compute distribution statistics (min, max, mean, stddev, percentiles) - Analyze overlap between TP and FP distributions - Suggest optimal threshold based on data - Show per-detection breakdown with top-5 matches - Create analyze_similarity_distribution.sh wrapper script - Supports baseline, finetuned, or both models - Saves output to similarity_analysis/ directory
This commit is contained in:
Rick McEwen
@ -265,6 +265,11 @@ def main():
|
||||
action="store_true",
|
||||
help="Enable verbose logging",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--similarity-details",
|
||||
action="store_true",
|
||||
help="Output detailed similarity scores for each detection (for analyzing score distributions)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-cache",
|
||||
action="store_true",
|
||||
@ -411,6 +416,16 @@ def main():
|
||||
# Detailed results for analysis
|
||||
results = []
|
||||
|
||||
# Similarity distribution tracking (for --similarity-details)
|
||||
similarity_details = {
|
||||
"true_positive_sims": [], # Similarities for correct matches
|
||||
"false_positive_sims": [], # Similarities for wrong matches
|
||||
"missed_best_sims": [], # Best similarity for logos that should have matched but didn't
|
||||
"all_positive_sims": [], # All similarities between detected regions and correct logos
|
||||
"all_negative_sims": [], # All similarities between detected regions and wrong logos
|
||||
"detection_details": [], # Per-detection breakdown
|
||||
}
|
||||
|
||||
# Process test images
|
||||
for test_filename in tqdm(test_images, desc="Testing"):
|
||||
test_path = test_images_dir / test_filename
|
||||
@ -445,7 +460,38 @@ def main():
|
||||
|
||||
# Match detections against references using selected method
|
||||
matched_logos: Set[str] = set()
|
||||
for detection in detections:
|
||||
for det_idx, detection in enumerate(detections):
|
||||
# Compute similarities to all reference logos for detailed analysis
|
||||
if args.similarity_details:
|
||||
all_sims = {}
|
||||
for logo_name, ref_emb_list in multi_ref_embeddings.items():
|
||||
sims = []
|
||||
for ref_emb in ref_emb_list:
|
||||
sim = detector.compare_embeddings(detection["embedding"], ref_emb)
|
||||
sims.append(sim)
|
||||
# Use mean or max based on setting
|
||||
if args.use_max_similarity:
|
||||
all_sims[logo_name] = max(sims) if sims else 0
|
||||
else:
|
||||
all_sims[logo_name] = sum(sims) / len(sims) if sims else 0
|
||||
|
||||
# Track positive vs negative similarities
|
||||
for sim in sims:
|
||||
if logo_name in expected_logos:
|
||||
similarity_details["all_positive_sims"].append(sim)
|
||||
else:
|
||||
similarity_details["all_negative_sims"].append(sim)
|
||||
|
||||
# Store detection details
|
||||
sorted_sims = sorted(all_sims.items(), key=lambda x: -x[1])
|
||||
similarity_details["detection_details"].append({
|
||||
"image": test_filename,
|
||||
"detection_idx": det_idx,
|
||||
"expected_logos": list(expected_logos),
|
||||
"top_5_matches": sorted_sims[:5],
|
||||
"detr_score": detection.get("score", 0),
|
||||
})
|
||||
|
||||
if args.matching_method == "simple":
|
||||
# Simple matching: return ALL logos above threshold
|
||||
all_matches = detector.find_all_matches(
|
||||
@ -457,16 +503,21 @@ def main():
|
||||
matched_logos.add(label)
|
||||
|
||||
# Check if this is a correct match
|
||||
if label in expected_logos:
|
||||
is_correct = label in expected_logos
|
||||
if is_correct:
|
||||
true_positives += 1
|
||||
if args.similarity_details:
|
||||
similarity_details["true_positive_sims"].append(similarity)
|
||||
else:
|
||||
false_positives += 1
|
||||
if args.similarity_details:
|
||||
similarity_details["false_positive_sims"].append(similarity)
|
||||
|
||||
results.append({
|
||||
"test_image": test_filename,
|
||||
"matched_logo": label,
|
||||
"similarity": similarity,
|
||||
"correct": label in expected_logos,
|
||||
"correct": is_correct,
|
||||
})
|
||||
|
||||
elif args.matching_method == "margin":
|
||||
@ -481,16 +532,21 @@ def main():
|
||||
label, similarity = match_result
|
||||
matched_logos.add(label)
|
||||
|
||||
if label in expected_logos:
|
||||
is_correct = label in expected_logos
|
||||
if is_correct:
|
||||
true_positives += 1
|
||||
if args.similarity_details:
|
||||
similarity_details["true_positive_sims"].append(similarity)
|
||||
else:
|
||||
false_positives += 1
|
||||
if args.similarity_details:
|
||||
similarity_details["false_positive_sims"].append(similarity)
|
||||
|
||||
results.append({
|
||||
"test_image": test_filename,
|
||||
"matched_logo": label,
|
||||
"similarity": similarity,
|
||||
"correct": label in expected_logos,
|
||||
"correct": is_correct,
|
||||
})
|
||||
|
||||
else: # multi-ref
|
||||
@ -507,16 +563,21 @@ def main():
|
||||
label, similarity, num_matching = match_result
|
||||
matched_logos.add(label)
|
||||
|
||||
if label in expected_logos:
|
||||
is_correct = label in expected_logos
|
||||
if is_correct:
|
||||
true_positives += 1
|
||||
if args.similarity_details:
|
||||
similarity_details["true_positive_sims"].append(similarity)
|
||||
else:
|
||||
false_positives += 1
|
||||
if args.similarity_details:
|
||||
similarity_details["false_positive_sims"].append(similarity)
|
||||
|
||||
results.append({
|
||||
"test_image": test_filename,
|
||||
"matched_logo": label,
|
||||
"similarity": similarity,
|
||||
"correct": label in expected_logos,
|
||||
"correct": is_correct,
|
||||
})
|
||||
|
||||
# Count missed detections (false negatives)
|
||||
@ -524,6 +585,15 @@ def main():
|
||||
false_negatives += len(missed)
|
||||
|
||||
for missed_logo in missed:
|
||||
# Track best similarity for missed logos (if we have detections)
|
||||
if args.similarity_details and detections:
|
||||
best_sim_for_missed = 0
|
||||
for detection in detections:
|
||||
for ref_emb in multi_ref_embeddings.get(missed_logo, []):
|
||||
sim = detector.compare_embeddings(detection["embedding"], ref_emb)
|
||||
best_sim_for_missed = max(best_sim_for_missed, sim)
|
||||
similarity_details["missed_best_sims"].append(best_sim_for_missed)
|
||||
|
||||
results.append({
|
||||
"test_image": test_filename,
|
||||
"matched_logo": None,
|
||||
@ -593,6 +663,10 @@ def main():
|
||||
|
||||
print("=" * 60)
|
||||
|
||||
# Print similarity distribution details if requested
|
||||
if args.similarity_details:
|
||||
print_similarity_details(similarity_details, args.threshold)
|
||||
|
||||
# Write results to file if requested
|
||||
if args.output_file:
|
||||
write_results_to_file(
|
||||
@ -612,6 +686,116 @@ def main():
|
||||
print(f"\nResults appended to: {args.output_file}")
|
||||
|
||||
|
||||
def print_similarity_details(details: dict, threshold: float):
|
||||
"""Print detailed similarity distribution analysis."""
|
||||
import statistics
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("SIMILARITY DISTRIBUTION ANALYSIS")
|
||||
print("=" * 60)
|
||||
|
||||
# Helper to compute stats
|
||||
def compute_stats(values, name):
|
||||
if not values:
|
||||
print(f"\n{name}: No data")
|
||||
return
|
||||
print(f"\n{name} (n={len(values)}):")
|
||||
print(f" Min: {min(values):.4f}")
|
||||
print(f" Max: {max(values):.4f}")
|
||||
print(f" Mean: {statistics.mean(values):.4f}")
|
||||
if len(values) > 1:
|
||||
print(f" StdDev: {statistics.stdev(values):.4f}")
|
||||
print(f" Median: {statistics.median(values):.4f}")
|
||||
|
||||
# Percentiles
|
||||
sorted_vals = sorted(values)
|
||||
n = len(sorted_vals)
|
||||
p10 = sorted_vals[int(n * 0.10)] if n > 10 else sorted_vals[0]
|
||||
p25 = sorted_vals[int(n * 0.25)] if n > 4 else sorted_vals[0]
|
||||
p75 = sorted_vals[int(n * 0.75)] if n > 4 else sorted_vals[-1]
|
||||
p90 = sorted_vals[int(n * 0.90)] if n > 10 else sorted_vals[-1]
|
||||
print(f" P10: {p10:.4f}")
|
||||
print(f" P25: {p25:.4f}")
|
||||
print(f" P75: {p75:.4f}")
|
||||
print(f" P90: {p90:.4f}")
|
||||
|
||||
# Count above/below threshold
|
||||
above = sum(1 for v in values if v >= threshold)
|
||||
below = sum(1 for v in values if v < threshold)
|
||||
print(f" Above threshold ({threshold}): {above} ({100*above/len(values):.1f}%)")
|
||||
print(f" Below threshold ({threshold}): {below} ({100*below/len(values):.1f}%)")
|
||||
|
||||
# Print distribution stats
|
||||
compute_stats(details["true_positive_sims"], "TRUE POSITIVE similarities (correct matches)")
|
||||
compute_stats(details["false_positive_sims"], "FALSE POSITIVE similarities (wrong matches)")
|
||||
compute_stats(details["missed_best_sims"], "MISSED LOGO best similarities (false negatives)")
|
||||
compute_stats(details["all_positive_sims"], "ALL similarities to CORRECT logos (per-ref)")
|
||||
compute_stats(details["all_negative_sims"], "ALL similarities to WRONG logos (per-ref)")
|
||||
|
||||
# Overlap analysis
|
||||
tp_sims = details["true_positive_sims"]
|
||||
fp_sims = details["false_positive_sims"]
|
||||
if tp_sims and fp_sims:
|
||||
print("\n" + "-" * 40)
|
||||
print("OVERLAP ANALYSIS:")
|
||||
tp_min, tp_max = min(tp_sims), max(tp_sims)
|
||||
fp_min, fp_max = min(fp_sims), max(fp_sims)
|
||||
print(f" True Positives range: [{tp_min:.4f}, {tp_max:.4f}]")
|
||||
print(f" False Positives range: [{fp_min:.4f}, {fp_max:.4f}]")
|
||||
|
||||
# Check overlap
|
||||
overlap_min = max(tp_min, fp_min)
|
||||
overlap_max = min(tp_max, fp_max)
|
||||
if overlap_min < overlap_max:
|
||||
print(f" OVERLAP REGION: [{overlap_min:.4f}, {overlap_max:.4f}]")
|
||||
tp_in_overlap = sum(1 for v in tp_sims if overlap_min <= v <= overlap_max)
|
||||
fp_in_overlap = sum(1 for v in fp_sims if overlap_min <= v <= overlap_max)
|
||||
print(f" TPs in overlap: {tp_in_overlap} ({100*tp_in_overlap/len(tp_sims):.1f}%)")
|
||||
print(f" FPs in overlap: {fp_in_overlap} ({100*fp_in_overlap/len(fp_sims):.1f}%)")
|
||||
else:
|
||||
print(" NO OVERLAP - distributions are separable!")
|
||||
|
||||
# Suggest optimal threshold
|
||||
all_points = [(s, "tp") for s in tp_sims] + [(s, "fp") for s in fp_sims]
|
||||
all_points.sort()
|
||||
best_thresh = threshold
|
||||
best_f1 = 0
|
||||
total_tp = len(tp_sims)
|
||||
total_fp = len(fp_sims)
|
||||
|
||||
for thresh in [p[0] for p in all_points]:
|
||||
# At this threshold:
|
||||
tp_above = sum(1 for s in tp_sims if s >= thresh)
|
||||
fp_above = sum(1 for s in fp_sims if s >= thresh)
|
||||
prec = tp_above / (tp_above + fp_above) if (tp_above + fp_above) > 0 else 0
|
||||
rec = tp_above / total_tp if total_tp > 0 else 0
|
||||
f1 = 2 * prec * rec / (prec + rec) if (prec + rec) > 0 else 0
|
||||
if f1 > best_f1:
|
||||
best_f1 = f1
|
||||
best_thresh = thresh
|
||||
|
||||
print(f"\n SUGGESTED OPTIMAL THRESHOLD: {best_thresh:.4f}")
|
||||
print(f" (would give F1 = {best_f1:.4f} on this data)")
|
||||
|
||||
# Print sample detection details
|
||||
det_details = details["detection_details"]
|
||||
if det_details:
|
||||
print("\n" + "-" * 40)
|
||||
print(f"SAMPLE DETECTION DETAILS (first 20 of {len(det_details)}):")
|
||||
for i, det in enumerate(det_details[:20]):
|
||||
expected = det["expected_logos"]
|
||||
top5 = det["top_5_matches"]
|
||||
print(f"\n [{i+1}] Image: {det['image']}")
|
||||
print(f" Expected: {expected if expected else '(none)'}")
|
||||
print(f" DETR score: {det['detr_score']:.3f}")
|
||||
print(f" Top 5 matches:")
|
||||
for logo, sim in top5:
|
||||
marker = " <-- CORRECT" if logo in expected else ""
|
||||
print(f" {sim:.4f} {logo}{marker}")
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
|
||||
|
||||
def write_results_to_file(
|
||||
output_path: Path,
|
||||
args,
|
||||
|
||||
Reference in New Issue
Block a user