Add threshold test script for image-split model

2026-01-07 10:14:21 -05:00
parent 569285f664
commit 2f28aa6052
1 changed files with 181 additions and 0 deletions
--- a/run_threshold_tests_image_split.sh
+++ b/run_threshold_tests_image_split.sh
@ -0,0 +1,181 @@
+#!/bin/bash
+#
+# Run logo detection tests with the image-split fine-tuned model.
+# Tests various threshold and margin settings to find optimal parameters.
+#
+# Usage:
+#   ./run_threshold_tests_image_split.sh
+#
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+OUTPUT_FILE="${SCRIPT_DIR}/threshold_test_results_image_split.txt"
+
+# Model path
+MODEL_PATH="models/logo_detection/clip_finetuned_image_split"
+
+# Common parameters
+NUM_LOGOS=20
+REFS_PER_LOGO=10
+POSITIVE_SAMPLES=20
+NEGATIVE_SAMPLES=100
+MIN_MATCHING_REFS=3
+SEED=42
+
+# Check if model exists
+if [ ! -d "${SCRIPT_DIR}/${MODEL_PATH}" ]; then
+    echo "Error: Image-split model not found at ${SCRIPT_DIR}/${MODEL_PATH}"
+    echo "Train the model first with: python train_clip_logo.py --config configs/cloud_rtx4090_image_split.yaml"
+    exit 1
+fi
+
+# Clear output file and write header
+echo "Threshold Optimization Tests - Image-Split Model" > "$OUTPUT_FILE"
+echo "=================================================" >> "$OUTPUT_FILE"
+echo "Date: $(date)" >> "$OUTPUT_FILE"
+echo "" >> "$OUTPUT_FILE"
+echo "Model: ${MODEL_PATH}" >> "$OUTPUT_FILE"
+echo "" >> "$OUTPUT_FILE"
+echo "Common Parameters:" >> "$OUTPUT_FILE"
+echo "  Matching method: multi-ref (max)" >> "$OUTPUT_FILE"
+echo "  Reference logos: $NUM_LOGOS" >> "$OUTPUT_FILE"
+echo "  Refs per logo: $REFS_PER_LOGO" >> "$OUTPUT_FILE"
+echo "  Positive samples: $POSITIVE_SAMPLES" >> "$OUTPUT_FILE"
+echo "  Negative samples: $NEGATIVE_SAMPLES" >> "$OUTPUT_FILE"
+echo "  Min matching refs: $MIN_MATCHING_REFS" >> "$OUTPUT_FILE"
+echo "  Seed: $SEED" >> "$OUTPUT_FILE"
+echo "" >> "$OUTPUT_FILE"
+
+echo "Running threshold optimization tests for image-split model..."
+echo "  Model: ${MODEL_PATH}"
+echo "  Matching method: multi-ref (max)"
+echo "  Reference logos: $NUM_LOGOS"
+echo "  Refs per logo: $REFS_PER_LOGO"
+echo "  Seed: $SEED"
+echo ""
+
+# Test 1: Lower threshold (image-split model may have different distribution)
+echo "=== Test 1: threshold=0.65, margin=0.05 ==="
+uv run python "$SCRIPT_DIR/test_logo_detection.py" \
+    --num-logos $NUM_LOGOS \
+    --refs-per-logo $REFS_PER_LOGO \
+    --positive-samples $POSITIVE_SAMPLES \
+    --negative-samples $NEGATIVE_SAMPLES \
+    --matching-method multi-ref \
+    --min-matching-refs $MIN_MATCHING_REFS \
+    --use-max-similarity \
+    --threshold 0.65 \
+    --margin 0.05 \
+    --seed $SEED \
+    --embedding-model "$MODEL_PATH" \
+    --output-file "$OUTPUT_FILE"
+
+echo ""
+
+# Test 2: Default threshold
+echo "=== Test 2: threshold=0.70, margin=0.05 ==="
+uv run python "$SCRIPT_DIR/test_logo_detection.py" \
+    --num-logos $NUM_LOGOS \
+    --refs-per-logo $REFS_PER_LOGO \
+    --positive-samples $POSITIVE_SAMPLES \
+    --negative-samples $NEGATIVE_SAMPLES \
+    --matching-method multi-ref \
+    --min-matching-refs $MIN_MATCHING_REFS \
+    --use-max-similarity \
+    --threshold 0.70 \
+    --margin 0.05 \
+    --seed $SEED \
+    --embedding-model "$MODEL_PATH" \
+    --output-file "$OUTPUT_FILE"
+
+echo ""
+
+# Test 3: threshold=0.75
+echo "=== Test 3: threshold=0.75, margin=0.05 ==="
+uv run python "$SCRIPT_DIR/test_logo_detection.py" \
+    --num-logos $NUM_LOGOS \
+    --refs-per-logo $REFS_PER_LOGO \
+    --positive-samples $POSITIVE_SAMPLES \
+    --negative-samples $NEGATIVE_SAMPLES \
+    --matching-method multi-ref \
+    --min-matching-refs $MIN_MATCHING_REFS \
+    --use-max-similarity \
+    --threshold 0.75 \
+    --margin 0.05 \
+    --seed $SEED \
+    --embedding-model "$MODEL_PATH" \
+    --output-file "$OUTPUT_FILE"
+
+echo ""
+
+# Test 4: threshold=0.80
+echo "=== Test 4: threshold=0.80, margin=0.05 ==="
+uv run python "$SCRIPT_DIR/test_logo_detection.py" \
+    --num-logos $NUM_LOGOS \
+    --refs-per-logo $REFS_PER_LOGO \
+    --positive-samples $POSITIVE_SAMPLES \
+    --negative-samples $NEGATIVE_SAMPLES \
+    --matching-method multi-ref \
+    --min-matching-refs $MIN_MATCHING_REFS \
+    --use-max-similarity \
+    --threshold 0.80 \
+    --margin 0.05 \
+    --seed $SEED \
+    --embedding-model "$MODEL_PATH" \
+    --output-file "$OUTPUT_FILE"
+
+echo ""
+
+# Test 5: threshold=0.80 with larger margin
+echo "=== Test 5: threshold=0.80, margin=0.10 ==="
+uv run python "$SCRIPT_DIR/test_logo_detection.py" \
+    --num-logos $NUM_LOGOS \
+    --refs-per-logo $REFS_PER_LOGO \
+    --positive-samples $POSITIVE_SAMPLES \
+    --negative-samples $NEGATIVE_SAMPLES \
+    --matching-method multi-ref \
+    --min-matching-refs $MIN_MATCHING_REFS \
+    --use-max-similarity \
+    --threshold 0.80 \
+    --margin 0.10 \
+    --seed $SEED \
+    --embedding-model "$MODEL_PATH" \
+    --output-file "$OUTPUT_FILE"
+
+echo ""
+
+# Test 6: threshold=0.85
+echo "=== Test 6: threshold=0.85, margin=0.10 ==="
+uv run python "$SCRIPT_DIR/test_logo_detection.py" \
+    --num-logos $NUM_LOGOS \
+    --refs-per-logo $REFS_PER_LOGO \
+    --positive-samples $POSITIVE_SAMPLES \
+    --negative-samples $NEGATIVE_SAMPLES \
+    --matching-method multi-ref \
+    --min-matching-refs $MIN_MATCHING_REFS \
+    --use-max-similarity \
+    --threshold 0.85 \
+    --margin 0.10 \
+    --seed $SEED \
+    --embedding-model "$MODEL_PATH" \
+    --output-file "$OUTPUT_FILE"
+
+echo ""
+
+# Test 7: threshold=0.90
+echo "=== Test 7: threshold=0.90, margin=0.10 ==="
+uv run python "$SCRIPT_DIR/test_logo_detection.py" \
+    --num-logos $NUM_LOGOS \
+    --refs-per-logo $REFS_PER_LOGO \
+    --positive-samples $POSITIVE_SAMPLES \
+    --negative-samples $NEGATIVE_SAMPLES \
+    --matching-method multi-ref \
+    --min-matching-refs $MIN_MATCHING_REFS \
+    --use-max-similarity \
+    --threshold 0.90 \
+    --margin 0.10 \
+    --seed $SEED \
+    --embedding-model "$MODEL_PATH" \
+    --output-file "$OUTPUT_FILE"
+
+echo ""
+echo "Results saved to: $OUTPUT_FILE"