logo_test/run_threshold_tests_image_split.sh

#!/bin/bash
#
# Run logo detection tests with the image-split fine-tuned model.
# Tests various threshold and margin settings to find optimal parameters.
#
# Usage:
#   ./run_threshold_tests_image_split.sh
#

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
OUTPUT_FILE="${SCRIPT_DIR}/threshold_test_results_image_split.txt"

# Model path
MODEL_PATH="models/logo_detection/clip_finetuned_image_split"

# Common parameters
NUM_LOGOS=20
REFS_PER_LOGO=10
POSITIVE_SAMPLES=20
NEGATIVE_SAMPLES=100
MIN_MATCHING_REFS=3
SEED=42

# Check if model exists
if [ ! -d "${SCRIPT_DIR}/${MODEL_PATH}" ]; then
    echo "Error: Image-split model not found at ${SCRIPT_DIR}/${MODEL_PATH}"
    echo "Train the model first with: python train_clip_logo.py --config configs/cloud_rtx4090_image_split.yaml"
    exit 1
fi

# Clear output file and write header
echo "Threshold Optimization Tests - Image-Split Model" > "$OUTPUT_FILE"
echo "=================================================" >> "$OUTPUT_FILE"
echo "Date: $(date)" >> "$OUTPUT_FILE"
echo "" >> "$OUTPUT_FILE"
echo "Model: ${MODEL_PATH}" >> "$OUTPUT_FILE"
echo "" >> "$OUTPUT_FILE"
echo "Common Parameters:" >> "$OUTPUT_FILE"
echo "  Matching method: multi-ref (max)" >> "$OUTPUT_FILE"
echo "  Reference logos: $NUM_LOGOS" >> "$OUTPUT_FILE"
echo "  Refs per logo: $REFS_PER_LOGO" >> "$OUTPUT_FILE"
echo "  Positive samples: $POSITIVE_SAMPLES" >> "$OUTPUT_FILE"
echo "  Negative samples: $NEGATIVE_SAMPLES" >> "$OUTPUT_FILE"
echo "  Min matching refs: $MIN_MATCHING_REFS" >> "$OUTPUT_FILE"
echo "  Seed: $SEED" >> "$OUTPUT_FILE"
echo "" >> "$OUTPUT_FILE"

echo "Running threshold optimization tests for image-split model..."
echo "  Model: ${MODEL_PATH}"
echo "  Matching method: multi-ref (max)"
echo "  Reference logos: $NUM_LOGOS"
echo "  Refs per logo: $REFS_PER_LOGO"
echo "  Seed: $SEED"
echo ""

# Test 1: Lower threshold (image-split model may have different distribution)
echo "=== Test 1: threshold=0.65, margin=0.05 ==="
uv run python "$SCRIPT_DIR/test_logo_detection.py" \
    --num-logos $NUM_LOGOS \
    --refs-per-logo $REFS_PER_LOGO \
    --positive-samples $POSITIVE_SAMPLES \
    --negative-samples $NEGATIVE_SAMPLES \
    --matching-method multi-ref \
    --min-matching-refs $MIN_MATCHING_REFS \
    --use-max-similarity \
    --threshold 0.65 \
    --margin 0.05 \
    --seed $SEED \
    --embedding-model "$MODEL_PATH" \
    --output-file "$OUTPUT_FILE"

echo ""

# Test 2: Default threshold
echo "=== Test 2: threshold=0.70, margin=0.05 ==="
uv run python "$SCRIPT_DIR/test_logo_detection.py" \
    --num-logos $NUM_LOGOS \
    --refs-per-logo $REFS_PER_LOGO \
    --positive-samples $POSITIVE_SAMPLES \
    --negative-samples $NEGATIVE_SAMPLES \
    --matching-method multi-ref \
    --min-matching-refs $MIN_MATCHING_REFS \
    --use-max-similarity \
    --threshold 0.70 \
    --margin 0.05 \
    --seed $SEED \
    --embedding-model "$MODEL_PATH" \
    --output-file "$OUTPUT_FILE"

echo ""

# Test 3: threshold=0.75
echo "=== Test 3: threshold=0.75, margin=0.05 ==="
uv run python "$SCRIPT_DIR/test_logo_detection.py" \
    --num-logos $NUM_LOGOS \
    --refs-per-logo $REFS_PER_LOGO \
    --positive-samples $POSITIVE_SAMPLES \
    --negative-samples $NEGATIVE_SAMPLES \
    --matching-method multi-ref \
    --min-matching-refs $MIN_MATCHING_REFS \
    --use-max-similarity \
    --threshold 0.75 \
    --margin 0.05 \
    --seed $SEED \
    --embedding-model "$MODEL_PATH" \
    --output-file "$OUTPUT_FILE"

echo ""

# Test 4: threshold=0.80
echo "=== Test 4: threshold=0.80, margin=0.05 ==="
uv run python "$SCRIPT_DIR/test_logo_detection.py" \
    --num-logos $NUM_LOGOS \
    --refs-per-logo $REFS_PER_LOGO \
    --positive-samples $POSITIVE_SAMPLES \
    --negative-samples $NEGATIVE_SAMPLES \
    --matching-method multi-ref \
    --min-matching-refs $MIN_MATCHING_REFS \
    --use-max-similarity \
    --threshold 0.80 \
    --margin 0.05 \
    --seed $SEED \
    --embedding-model "$MODEL_PATH" \
    --output-file "$OUTPUT_FILE"

echo ""

# Test 5: threshold=0.80 with larger margin
echo "=== Test 5: threshold=0.80, margin=0.10 ==="
uv run python "$SCRIPT_DIR/test_logo_detection.py" \
    --num-logos $NUM_LOGOS \
    --refs-per-logo $REFS_PER_LOGO \
    --positive-samples $POSITIVE_SAMPLES \
    --negative-samples $NEGATIVE_SAMPLES \
    --matching-method multi-ref \
    --min-matching-refs $MIN_MATCHING_REFS \
    --use-max-similarity \
    --threshold 0.80 \
    --margin 0.10 \
    --seed $SEED \
    --embedding-model "$MODEL_PATH" \
    --output-file "$OUTPUT_FILE"

echo ""

# Test 6: threshold=0.85
echo "=== Test 6: threshold=0.85, margin=0.10 ==="
uv run python "$SCRIPT_DIR/test_logo_detection.py" \
    --num-logos $NUM_LOGOS \
    --refs-per-logo $REFS_PER_LOGO \
    --positive-samples $POSITIVE_SAMPLES \
    --negative-samples $NEGATIVE_SAMPLES \
    --matching-method multi-ref \
    --min-matching-refs $MIN_MATCHING_REFS \
    --use-max-similarity \
    --threshold 0.85 \
    --margin 0.10 \
    --seed $SEED \
    --embedding-model "$MODEL_PATH" \
    --output-file "$OUTPUT_FILE"

echo ""

# Test 7: threshold=0.90
echo "=== Test 7: threshold=0.90, margin=0.10 ==="
uv run python "$SCRIPT_DIR/test_logo_detection.py" \
    --num-logos $NUM_LOGOS \
    --refs-per-logo $REFS_PER_LOGO \
    --positive-samples $POSITIVE_SAMPLES \
    --negative-samples $NEGATIVE_SAMPLES \
    --matching-method multi-ref \
    --min-matching-refs $MIN_MATCHING_REFS \
    --use-max-similarity \
    --threshold 0.90 \
    --margin 0.10 \
    --seed $SEED \
    --embedding-model "$MODEL_PATH" \
    --output-file "$OUTPUT_FILE"

echo ""
echo "Results saved to: $OUTPUT_FILE"