Add script to test optimal refs per logo for baseline CLIP

2026-01-07 12:52:16 -05:00
parent b5432c9ef7
commit 78f46f04bf
1 changed files with 132 additions and 0 deletions
--- a/run_refs_per_logo_test.sh
+++ b/run_refs_per_logo_test.sh
@ -0,0 +1,132 @@
+#!/bin/bash
+#
+# Test different numbers of reference logos per brand to find optimal setting.
+# Uses baseline CLIP with multi-ref (max) matching method.
+#
+# Usage:
+#   ./run_refs_per_logo_test.sh
+#
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+OUTPUT_FILE="${SCRIPT_DIR}/test_results/refs_per_logo_analysis.txt"
+
+# Model - baseline CLIP (best for unknown logos)
+MODEL="openai/clip-vit-large-patch14"
+
+# Fixed parameters
+NUM_LOGOS=20
+POSITIVE_SAMPLES=20
+NEGATIVE_SAMPLES=100
+MIN_MATCHING_REFS=1
+THRESHOLD=0.70
+MARGIN=0.05
+SEED=42
+
+# Refs per logo values to test
+REFS_TO_TEST="1 2 3 5 7 10 15 20"
+
+# Create output directory if needed
+mkdir -p "${SCRIPT_DIR}/test_results"
+
+# Clear output file and write header
+cat > "$OUTPUT_FILE" << EOF
+Reference Logos Per Brand Optimization
+======================================
+Date: $(date)
+
+Model: ${MODEL}
+Method: multi-ref (max)
+
+Fixed Parameters:
+  Number of logo brands: ${NUM_LOGOS}
+  Similarity threshold: ${THRESHOLD}
+  Margin: ${MARGIN}
+  Min matching refs: ${MIN_MATCHING_REFS}
+  Positive samples/logo: ${POSITIVE_SAMPLES}
+  Negative samples/logo: ${NEGATIVE_SAMPLES}
+  Seed: ${SEED}
+
+Testing refs per logo: ${REFS_TO_TEST}
+
+EOF
+
+echo "Reference Logos Per Brand Optimization"
+echo "======================================="
+echo "Model: ${MODEL}"
+echo "Testing refs per logo: ${REFS_TO_TEST}"
+echo ""
+
+# Results table header
+echo "Results Summary:" >> "$OUTPUT_FILE"
+echo "----------------" >> "$OUTPUT_FILE"
+printf "%-12s %8s %8s %8s %8s %8s %8s\n" "Refs/Logo" "TP" "FP" "FN" "Prec" "Recall" "F1" >> "$OUTPUT_FILE"
+echo "------------------------------------------------------------------------" >> "$OUTPUT_FILE"
+
+# Track best result
+BEST_F1=0
+BEST_REFS=0
+
+for REFS in ${REFS_TO_TEST}; do
+    echo "=== Testing refs_per_logo=${REFS} ==="
+
+    # Run test and capture output
+    OUTPUT=$(uv run python "$SCRIPT_DIR/test_logo_detection.py" \
+        --num-logos $NUM_LOGOS \
+        --refs-per-logo $REFS \
+        --positive-samples $POSITIVE_SAMPLES \
+        --negative-samples $NEGATIVE_SAMPLES \
+        --matching-method multi-ref \
+        --min-matching-refs $MIN_MATCHING_REFS \
+        --use-max-similarity \
+        --threshold $THRESHOLD \
+        --margin $MARGIN \
+        --seed $SEED \
+        --embedding-model "$MODEL" \
+        2>&1)
+
+    # Extract metrics
+    TP=$(echo "${OUTPUT}" | grep "True Positives" | grep -oE "[0-9]+" | head -1)
+    FP=$(echo "${OUTPUT}" | grep "False Positives" | grep -oE "[0-9]+" | head -1)
+    FN=$(echo "${OUTPUT}" | grep "False Negatives" | grep -oE "[0-9]+" | head -1)
+    PREC=$(echo "${OUTPUT}" | grep "Precision:" | grep -oE "[0-9]+\.[0-9]+%" | head -1)
+    RECALL=$(echo "${OUTPUT}" | grep "Recall:" | grep -oE "[0-9]+\.[0-9]+%" | head -1)
+    F1=$(echo "${OUTPUT}" | grep "F1 Score:" | grep -oE "[0-9]+\.[0-9]+%" | head -1)
+
+    # Print to console
+    echo "  TP: ${TP}, FP: ${FP}, FN: ${FN}"
+    echo "  Precision: ${PREC}, Recall: ${RECALL}, F1: ${F1}"
+    echo ""
+
+    # Add to results table
+    printf "%-12s %8s %8s %8s %8s %8s %8s\n" "${REFS}" "${TP}" "${FP}" "${FN}" "${PREC}" "${RECALL}" "${F1}" >> "$OUTPUT_FILE"
+
+    # Track best F1
+    F1_NUM=$(echo "${F1}" | tr -d '%')
+    if [ -n "$F1_NUM" ]; then
+        BETTER=$(echo "${F1_NUM} > ${BEST_F1}" | bc -l 2>/dev/null || echo "0")
+        if [ "$BETTER" = "1" ]; then
+            BEST_F1="${F1_NUM}"
+            BEST_REFS="${REFS}"
+        fi
+    fi
+
+    # Also append full output for this test
+    echo "" >> "$OUTPUT_FILE"
+    echo "======================================================================" >> "$OUTPUT_FILE"
+    echo "DETAILED RESULTS: refs_per_logo=${REFS}" >> "$OUTPUT_FILE"
+    echo "======================================================================" >> "$OUTPUT_FILE"
+    echo "${OUTPUT}" | grep -A 50 "Configuration:" | head -30 >> "$OUTPUT_FILE"
+    echo "" >> "$OUTPUT_FILE"
+done
+
+# Summary
+echo "------------------------------------------------------------------------" >> "$OUTPUT_FILE"
+echo "" >> "$OUTPUT_FILE"
+echo "OPTIMAL SETTING: refs_per_logo=${BEST_REFS} (F1 = ${BEST_F1}%)" >> "$OUTPUT_FILE"
+echo "" >> "$OUTPUT_FILE"
+
+echo "======================================="
+echo "OPTIMAL: refs_per_logo=${BEST_REFS} (F1 = ${BEST_F1}%)"
+echo "======================================="
+echo ""
+echo "Results saved to: $OUTPUT_FILE"