diff --git a/run_threshold_tests_image_split.sh b/run_threshold_tests_image_split.sh new file mode 100755 index 0000000..6f37526 --- /dev/null +++ b/run_threshold_tests_image_split.sh @@ -0,0 +1,181 @@ +#!/bin/bash +# +# Run logo detection tests with the image-split fine-tuned model. +# Tests various threshold and margin settings to find optimal parameters. +# +# Usage: +# ./run_threshold_tests_image_split.sh +# + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OUTPUT_FILE="${SCRIPT_DIR}/threshold_test_results_image_split.txt" + +# Model path +MODEL_PATH="models/logo_detection/clip_finetuned_image_split" + +# Common parameters +NUM_LOGOS=20 +REFS_PER_LOGO=10 +POSITIVE_SAMPLES=20 +NEGATIVE_SAMPLES=100 +MIN_MATCHING_REFS=3 +SEED=42 + +# Check if model exists +if [ ! -d "${SCRIPT_DIR}/${MODEL_PATH}" ]; then + echo "Error: Image-split model not found at ${SCRIPT_DIR}/${MODEL_PATH}" + echo "Train the model first with: python train_clip_logo.py --config configs/cloud_rtx4090_image_split.yaml" + exit 1 +fi + +# Clear output file and write header +echo "Threshold Optimization Tests - Image-Split Model" > "$OUTPUT_FILE" +echo "=================================================" >> "$OUTPUT_FILE" +echo "Date: $(date)" >> "$OUTPUT_FILE" +echo "" >> "$OUTPUT_FILE" +echo "Model: ${MODEL_PATH}" >> "$OUTPUT_FILE" +echo "" >> "$OUTPUT_FILE" +echo "Common Parameters:" >> "$OUTPUT_FILE" +echo " Matching method: multi-ref (max)" >> "$OUTPUT_FILE" +echo " Reference logos: $NUM_LOGOS" >> "$OUTPUT_FILE" +echo " Refs per logo: $REFS_PER_LOGO" >> "$OUTPUT_FILE" +echo " Positive samples: $POSITIVE_SAMPLES" >> "$OUTPUT_FILE" +echo " Negative samples: $NEGATIVE_SAMPLES" >> "$OUTPUT_FILE" +echo " Min matching refs: $MIN_MATCHING_REFS" >> "$OUTPUT_FILE" +echo " Seed: $SEED" >> "$OUTPUT_FILE" +echo "" >> "$OUTPUT_FILE" + +echo "Running threshold optimization tests for image-split model..." +echo " Model: ${MODEL_PATH}" +echo " Matching method: multi-ref (max)" +echo " Reference logos: $NUM_LOGOS" +echo " Refs per logo: $REFS_PER_LOGO" +echo " Seed: $SEED" +echo "" + +# Test 1: Lower threshold (image-split model may have different distribution) +echo "=== Test 1: threshold=0.65, margin=0.05 ===" +uv run python "$SCRIPT_DIR/test_logo_detection.py" \ + --num-logos $NUM_LOGOS \ + --refs-per-logo $REFS_PER_LOGO \ + --positive-samples $POSITIVE_SAMPLES \ + --negative-samples $NEGATIVE_SAMPLES \ + --matching-method multi-ref \ + --min-matching-refs $MIN_MATCHING_REFS \ + --use-max-similarity \ + --threshold 0.65 \ + --margin 0.05 \ + --seed $SEED \ + --embedding-model "$MODEL_PATH" \ + --output-file "$OUTPUT_FILE" + +echo "" + +# Test 2: Default threshold +echo "=== Test 2: threshold=0.70, margin=0.05 ===" +uv run python "$SCRIPT_DIR/test_logo_detection.py" \ + --num-logos $NUM_LOGOS \ + --refs-per-logo $REFS_PER_LOGO \ + --positive-samples $POSITIVE_SAMPLES \ + --negative-samples $NEGATIVE_SAMPLES \ + --matching-method multi-ref \ + --min-matching-refs $MIN_MATCHING_REFS \ + --use-max-similarity \ + --threshold 0.70 \ + --margin 0.05 \ + --seed $SEED \ + --embedding-model "$MODEL_PATH" \ + --output-file "$OUTPUT_FILE" + +echo "" + +# Test 3: threshold=0.75 +echo "=== Test 3: threshold=0.75, margin=0.05 ===" +uv run python "$SCRIPT_DIR/test_logo_detection.py" \ + --num-logos $NUM_LOGOS \ + --refs-per-logo $REFS_PER_LOGO \ + --positive-samples $POSITIVE_SAMPLES \ + --negative-samples $NEGATIVE_SAMPLES \ + --matching-method multi-ref \ + --min-matching-refs $MIN_MATCHING_REFS \ + --use-max-similarity \ + --threshold 0.75 \ + --margin 0.05 \ + --seed $SEED \ + --embedding-model "$MODEL_PATH" \ + --output-file "$OUTPUT_FILE" + +echo "" + +# Test 4: threshold=0.80 +echo "=== Test 4: threshold=0.80, margin=0.05 ===" +uv run python "$SCRIPT_DIR/test_logo_detection.py" \ + --num-logos $NUM_LOGOS \ + --refs-per-logo $REFS_PER_LOGO \ + --positive-samples $POSITIVE_SAMPLES \ + --negative-samples $NEGATIVE_SAMPLES \ + --matching-method multi-ref \ + --min-matching-refs $MIN_MATCHING_REFS \ + --use-max-similarity \ + --threshold 0.80 \ + --margin 0.05 \ + --seed $SEED \ + --embedding-model "$MODEL_PATH" \ + --output-file "$OUTPUT_FILE" + +echo "" + +# Test 5: threshold=0.80 with larger margin +echo "=== Test 5: threshold=0.80, margin=0.10 ===" +uv run python "$SCRIPT_DIR/test_logo_detection.py" \ + --num-logos $NUM_LOGOS \ + --refs-per-logo $REFS_PER_LOGO \ + --positive-samples $POSITIVE_SAMPLES \ + --negative-samples $NEGATIVE_SAMPLES \ + --matching-method multi-ref \ + --min-matching-refs $MIN_MATCHING_REFS \ + --use-max-similarity \ + --threshold 0.80 \ + --margin 0.10 \ + --seed $SEED \ + --embedding-model "$MODEL_PATH" \ + --output-file "$OUTPUT_FILE" + +echo "" + +# Test 6: threshold=0.85 +echo "=== Test 6: threshold=0.85, margin=0.10 ===" +uv run python "$SCRIPT_DIR/test_logo_detection.py" \ + --num-logos $NUM_LOGOS \ + --refs-per-logo $REFS_PER_LOGO \ + --positive-samples $POSITIVE_SAMPLES \ + --negative-samples $NEGATIVE_SAMPLES \ + --matching-method multi-ref \ + --min-matching-refs $MIN_MATCHING_REFS \ + --use-max-similarity \ + --threshold 0.85 \ + --margin 0.10 \ + --seed $SEED \ + --embedding-model "$MODEL_PATH" \ + --output-file "$OUTPUT_FILE" + +echo "" + +# Test 7: threshold=0.90 +echo "=== Test 7: threshold=0.90, margin=0.10 ===" +uv run python "$SCRIPT_DIR/test_logo_detection.py" \ + --num-logos $NUM_LOGOS \ + --refs-per-logo $REFS_PER_LOGO \ + --positive-samples $POSITIVE_SAMPLES \ + --negative-samples $NEGATIVE_SAMPLES \ + --matching-method multi-ref \ + --min-matching-refs $MIN_MATCHING_REFS \ + --use-max-similarity \ + --threshold 0.90 \ + --margin 0.10 \ + --seed $SEED \ + --embedding-model "$MODEL_PATH" \ + --output-file "$OUTPUT_FILE" + +echo "" +echo "Results saved to: $OUTPUT_FILE"