diff --git a/run_refs_per_logo_test.sh b/run_refs_per_logo_test.sh new file mode 100755 index 0000000..05a79a0 --- /dev/null +++ b/run_refs_per_logo_test.sh @@ -0,0 +1,132 @@ +#!/bin/bash +# +# Test different numbers of reference logos per brand to find optimal setting. +# Uses baseline CLIP with multi-ref (max) matching method. +# +# Usage: +# ./run_refs_per_logo_test.sh +# + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OUTPUT_FILE="${SCRIPT_DIR}/test_results/refs_per_logo_analysis.txt" + +# Model - baseline CLIP (best for unknown logos) +MODEL="openai/clip-vit-large-patch14" + +# Fixed parameters +NUM_LOGOS=20 +POSITIVE_SAMPLES=20 +NEGATIVE_SAMPLES=100 +MIN_MATCHING_REFS=1 +THRESHOLD=0.70 +MARGIN=0.05 +SEED=42 + +# Refs per logo values to test +REFS_TO_TEST="1 2 3 5 7 10 15 20" + +# Create output directory if needed +mkdir -p "${SCRIPT_DIR}/test_results" + +# Clear output file and write header +cat > "$OUTPUT_FILE" << EOF +Reference Logos Per Brand Optimization +====================================== +Date: $(date) + +Model: ${MODEL} +Method: multi-ref (max) + +Fixed Parameters: + Number of logo brands: ${NUM_LOGOS} + Similarity threshold: ${THRESHOLD} + Margin: ${MARGIN} + Min matching refs: ${MIN_MATCHING_REFS} + Positive samples/logo: ${POSITIVE_SAMPLES} + Negative samples/logo: ${NEGATIVE_SAMPLES} + Seed: ${SEED} + +Testing refs per logo: ${REFS_TO_TEST} + +EOF + +echo "Reference Logos Per Brand Optimization" +echo "=======================================" +echo "Model: ${MODEL}" +echo "Testing refs per logo: ${REFS_TO_TEST}" +echo "" + +# Results table header +echo "Results Summary:" >> "$OUTPUT_FILE" +echo "----------------" >> "$OUTPUT_FILE" +printf "%-12s %8s %8s %8s %8s %8s %8s\n" "Refs/Logo" "TP" "FP" "FN" "Prec" "Recall" "F1" >> "$OUTPUT_FILE" +echo "------------------------------------------------------------------------" >> "$OUTPUT_FILE" + +# Track best result +BEST_F1=0 +BEST_REFS=0 + +for REFS in ${REFS_TO_TEST}; do + echo "=== Testing refs_per_logo=${REFS} ===" + + # Run test and capture output + OUTPUT=$(uv run python "$SCRIPT_DIR/test_logo_detection.py" \ + --num-logos $NUM_LOGOS \ + --refs-per-logo $REFS \ + --positive-samples $POSITIVE_SAMPLES \ + --negative-samples $NEGATIVE_SAMPLES \ + --matching-method multi-ref \ + --min-matching-refs $MIN_MATCHING_REFS \ + --use-max-similarity \ + --threshold $THRESHOLD \ + --margin $MARGIN \ + --seed $SEED \ + --embedding-model "$MODEL" \ + 2>&1) + + # Extract metrics + TP=$(echo "${OUTPUT}" | grep "True Positives" | grep -oE "[0-9]+" | head -1) + FP=$(echo "${OUTPUT}" | grep "False Positives" | grep -oE "[0-9]+" | head -1) + FN=$(echo "${OUTPUT}" | grep "False Negatives" | grep -oE "[0-9]+" | head -1) + PREC=$(echo "${OUTPUT}" | grep "Precision:" | grep -oE "[0-9]+\.[0-9]+%" | head -1) + RECALL=$(echo "${OUTPUT}" | grep "Recall:" | grep -oE "[0-9]+\.[0-9]+%" | head -1) + F1=$(echo "${OUTPUT}" | grep "F1 Score:" | grep -oE "[0-9]+\.[0-9]+%" | head -1) + + # Print to console + echo " TP: ${TP}, FP: ${FP}, FN: ${FN}" + echo " Precision: ${PREC}, Recall: ${RECALL}, F1: ${F1}" + echo "" + + # Add to results table + printf "%-12s %8s %8s %8s %8s %8s %8s\n" "${REFS}" "${TP}" "${FP}" "${FN}" "${PREC}" "${RECALL}" "${F1}" >> "$OUTPUT_FILE" + + # Track best F1 + F1_NUM=$(echo "${F1}" | tr -d '%') + if [ -n "$F1_NUM" ]; then + BETTER=$(echo "${F1_NUM} > ${BEST_F1}" | bc -l 2>/dev/null || echo "0") + if [ "$BETTER" = "1" ]; then + BEST_F1="${F1_NUM}" + BEST_REFS="${REFS}" + fi + fi + + # Also append full output for this test + echo "" >> "$OUTPUT_FILE" + echo "======================================================================" >> "$OUTPUT_FILE" + echo "DETAILED RESULTS: refs_per_logo=${REFS}" >> "$OUTPUT_FILE" + echo "======================================================================" >> "$OUTPUT_FILE" + echo "${OUTPUT}" | grep -A 50 "Configuration:" | head -30 >> "$OUTPUT_FILE" + echo "" >> "$OUTPUT_FILE" +done + +# Summary +echo "------------------------------------------------------------------------" >> "$OUTPUT_FILE" +echo "" >> "$OUTPUT_FILE" +echo "OPTIMAL SETTING: refs_per_logo=${BEST_REFS} (F1 = ${BEST_F1}%)" >> "$OUTPUT_FILE" +echo "" >> "$OUTPUT_FILE" + +echo "=======================================" +echo "OPTIMAL: refs_per_logo=${BEST_REFS} (F1 = ${BEST_F1}%)" +echo "=======================================" +echo "" +echo "Results saved to: $OUTPUT_FILE"