From 41bc0c701f3dd2561107936dd427b4ef05bf6058 Mon Sep 17 00:00:00 2001 From: Rick McEwen Date: Wed, 31 Dec 2025 17:36:18 -0500 Subject: [PATCH] Add simple matching method as baseline for comparison tests - Add find_all_matches() method to DetectLogosDETR that returns all logos above similarity threshold without any rejection logic - Add --matching-method simple option to test script - Update run_comparison_tests.sh to include simple matching as Test 1 - Update documentation to describe simple matching method --- logo_detection_detr.py | 41 ++++++++++++++++ logo_detection_detr_usage.md | 35 ++++++++++++++ logo_detection_test_methodology.md | 35 +++++++++----- run_comparison_tests.sh | 28 ++++++++--- test_logo_detection.py | 75 +++++++++++++++++++++--------- 5 files changed, 174 insertions(+), 40 deletions(-) diff --git a/logo_detection_detr.py b/logo_detection_detr.py index fe94a30..fa14087 100644 --- a/logo_detection_detr.py +++ b/logo_detection_detr.py @@ -394,6 +394,47 @@ class DetectLogosDETR: else: return None + def find_all_matches( + self, + detected_embedding: torch.Tensor, + reference_embeddings: List[Tuple[str, torch.Tensor]], + similarity_threshold: float = 0.7, + ) -> List[Tuple[str, float]]: + """ + Find all matching reference logos above the similarity threshold. + + Unlike find_best_match, this returns ALL logos that have at least one + reference above threshold. Each unique logo is returned once with its + highest similarity score. + + Args: + detected_embedding: CLIP embedding from detected logo region + reference_embeddings: List of (label, embedding) tuples for reference logos + similarity_threshold: Minimum similarity to consider a match (0-1) + + Returns: + List of (label, similarity) tuples for all matches above threshold, + sorted by similarity descending. Each logo appears at most once. + """ + if not reference_embeddings: + return [] + + # Track best similarity for each logo + logo_best_sim: Dict[str, float] = {} + + for label, ref_embedding in reference_embeddings: + similarity = self.compare_embeddings(detected_embedding, ref_embedding) + + if similarity >= similarity_threshold: + if label not in logo_best_sim or similarity > logo_best_sim[label]: + logo_best_sim[label] = similarity + + # Convert to list and sort by similarity descending + matches = [(label, sim) for label, sim in logo_best_sim.items()] + matches.sort(key=lambda x: x[1], reverse=True) + + return matches + def find_best_match_multi_ref( self, detected_embedding: torch.Tensor, diff --git a/logo_detection_detr_usage.md b/logo_detection_detr_usage.md index e17e475..85bfd35 100644 --- a/logo_detection_detr_usage.md +++ b/logo_detection_detr_usage.md @@ -78,6 +78,41 @@ match = detector.find_best_match( **Returns:** - Tuple of (label, similarity) for best match, or None if no match above threshold +#### `find_all_matches()` - Find all matching reference logos + +Returns ALL logos that have at least one reference above the similarity threshold. Each unique logo appears once with its highest similarity score. + +```python +matches = detector.find_all_matches( + detected_embedding, + reference_embeddings, + similarity_threshold=0.7 +) +# Returns: [(label1, similarity1), (label2, similarity2), ...] +``` + +**Parameters:** +- `detected_embedding`: CLIP embedding from detected logo region +- `reference_embeddings`: List of (label, embedding) tuples for reference logos +- `similarity_threshold`: Minimum similarity to consider a match (0-1, default: 0.7) + +**Returns:** +- List of (label, similarity) tuples for all matches above threshold, sorted by similarity descending +- Each logo appears at most once (with its highest matching reference) + +**Example:** +```python +# Get all logos that match a detection +all_matches = detector.find_all_matches( + detection["embedding"], + reference_embeddings, + similarity_threshold=0.7 +) + +for label, similarity in all_matches: + print(f"Matched: {label} (similarity: {similarity:.3f})") +``` + #### `detect_and_match()` - One-step detection and matching ```python diff --git a/logo_detection_test_methodology.md b/logo_detection_test_methodology.md index 7621b6f..7dd3263 100644 --- a/logo_detection_test_methodology.md +++ b/logo_detection_test_methodology.md @@ -39,8 +39,8 @@ The system uses a two-stage pipeline: | Parameter | Default | Description | |-----------|---------|-------------| -| `--matching-method` | margin | Matching method: `margin` or `multi-ref` | -| `--margin` | 0.05 | Required margin between best and second-best match (applies to both methods) | +| `--matching-method` | margin | Matching method: `simple`, `margin`, or `multi-ref` | +| `--margin` | 0.05 | Required margin between best and second-best match (applies to `margin` and `multi-ref`) | #### Multi-Ref Method Parameters (when `--matching-method multi-ref`) @@ -193,11 +193,11 @@ This ensures cosine similarity is computed correctly and scores fall in the rang | Method | Test Script Option | Key Feature | |--------|-------------------|-------------| -| `find_best_match` | N/A (library only) | Returns highest similarity above threshold | +| `find_all_matches` | `--matching-method simple` | Returns ALL logos above threshold (baseline, most permissive) | | `find_best_match_with_margin` | `--matching-method margin` | Requires margin over second-best match | | `find_best_match_multi_ref` | `--matching-method multi-ref` | Aggregates scores across reference images | -The test script supports both `margin` and `multi-ref` matching methods via the `--matching-method` parameter. +The test script supports `simple`, `margin`, and `multi-ref` matching methods via the `--matching-method` parameter. --- @@ -242,13 +242,14 @@ Input Image ▼ ┌─────────────────────────────────────┐ │ Matching (selectable method) │ -│ ┌───────────────┬────────────────┐ │ -│ │ margin │ multi-ref │ │ -│ ├───────────────┼────────────────┤ │ -│ │ Require margin│ Aggregate │ │ -│ │ over 2nd best │ across refs │ │ -│ │ match │ (mean or max) │ │ -│ └───────────────┴────────────────┘ │ +│ ┌─────────┬─────────┬────────────┐ │ +│ │ simple │ margin │ multi-ref │ │ +│ ├─────────┼─────────┼────────────┤ │ +│ │ All │ Require │ Aggregate │ │ +│ │ matches │ margin │ across │ │ +│ │ above │ over │ refs │ │ +│ │ thresh │ 2nd best│ (mean/max) │ │ +│ └─────────┴─────────┴────────────┘ │ └─────────────────────────────────────┘ │ ▼ @@ -259,6 +260,15 @@ Matched Logo Labels ## Tuning Recommendations +### For Simple Matching (`--matching-method simple`) + +| Goal | Adjustments | +|------|-------------| +| **Reduce false positives** | Increase `--threshold` (only tuning option for simple method) | +| **Reduce false negatives** | Decrease `--threshold` | + +Note: Simple matching is primarily used as a baseline. For production use, consider `margin` or `multi-ref`. + ### For Margin-Based Matching (`--matching-method margin`) | Goal | Adjustments | @@ -287,6 +297,9 @@ Matched Logo Labels ## Example Usage ```bash +# Simple matching (baseline - all matches above threshold) +python test_logo_detection.py -n 20 --matching-method simple --threshold 0.70 + # Default margin-based matching python test_logo_detection.py -n 20 --threshold 0.75 --margin 0.05 diff --git a/run_comparison_tests.sh b/run_comparison_tests.sh index ce4c1da..8991880 100755 --- a/run_comparison_tests.sh +++ b/run_comparison_tests.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Run logo detection tests with all three matching methods and save results. +# Run logo detection tests with all four matching methods and save results. # SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -30,8 +30,22 @@ echo " Min matching refs: $MIN_MATCHING_REFS" echo " Seed: $SEED" echo "" -# Test 1: Margin-based matching -echo "=== Test 1: Margin-based matching ===" | tee -a "$OUTPUT_FILE" +# Test 1: Simple matching (baseline - all matches above threshold) +echo "=== Test 1: Simple matching (baseline) ===" | tee -a "$OUTPUT_FILE" +uv run python "$SCRIPT_DIR/test_logo_detection.py" \ + --num-logos $NUM_LOGOS \ + --refs-per-logo $REFS_PER_LOGO \ + --positive-samples $POSITIVE_SAMPLES \ + --negative-samples $NEGATIVE_SAMPLES \ + --matching-method simple \ + --seed $SEED \ + 2>&1 | tee -a "$OUTPUT_FILE" + +echo "" >> "$OUTPUT_FILE" +echo "" >> "$OUTPUT_FILE" + +# Test 2: Margin-based matching +echo "=== Test 2: Margin-based matching ===" | tee -a "$OUTPUT_FILE" uv run python "$SCRIPT_DIR/test_logo_detection.py" \ --num-logos $NUM_LOGOS \ --refs-per-logo $REFS_PER_LOGO \ @@ -44,8 +58,8 @@ uv run python "$SCRIPT_DIR/test_logo_detection.py" \ echo "" >> "$OUTPUT_FILE" echo "" >> "$OUTPUT_FILE" -# Test 2: Multi-ref with mean similarity -echo "=== Test 2: Multi-ref matching (mean similarity) ===" | tee -a "$OUTPUT_FILE" +# Test 3: Multi-ref with mean similarity +echo "=== Test 3: Multi-ref matching (mean similarity) ===" | tee -a "$OUTPUT_FILE" uv run python "$SCRIPT_DIR/test_logo_detection.py" \ --num-logos $NUM_LOGOS \ --refs-per-logo $REFS_PER_LOGO \ @@ -59,8 +73,8 @@ uv run python "$SCRIPT_DIR/test_logo_detection.py" \ echo "" >> "$OUTPUT_FILE" echo "" >> "$OUTPUT_FILE" -# Test 3: Multi-ref with max similarity -echo "=== Test 3: Multi-ref matching (max similarity) ===" | tee -a "$OUTPUT_FILE" +# Test 4: Multi-ref with max similarity +echo "=== Test 4: Multi-ref matching (max similarity) ===" | tee -a "$OUTPUT_FILE" uv run python "$SCRIPT_DIR/test_logo_detection.py" \ --num-logos $NUM_LOGOS \ --refs-per-logo $REFS_PER_LOGO \ diff --git a/test_logo_detection.py b/test_logo_detection.py index 65e4d60..a0b190a 100755 --- a/test_logo_detection.py +++ b/test_logo_detection.py @@ -236,9 +236,10 @@ def main(): parser.add_argument( "--matching-method", type=str, - choices=["margin", "multi-ref"], + choices=["simple", "margin", "multi-ref"], default="margin", - help="Matching method: 'margin' requires confidence margin over 2nd best, " + help="Matching method: 'simple' returns all matches above threshold, " + "'margin' requires confidence margin over 2nd best, " "'multi-ref' aggregates scores across reference images (default: margin)", ) parser.add_argument( @@ -431,10 +432,30 @@ def main(): # Match detections against references using selected method matched_logos: Set[str] = set() for detection in detections: - match = None - similarity = None + if args.matching_method == "simple": + # Simple matching: return ALL logos above threshold + all_matches = detector.find_all_matches( + detection["embedding"], + reference_embeddings, + similarity_threshold=args.threshold, + ) + for label, similarity in all_matches: + matched_logos.add(label) - if args.matching_method == "margin": + # Check if this is a correct match + if label in expected_logos: + true_positives += 1 + else: + false_positives += 1 + + results.append({ + "test_image": test_filename, + "matched_logo": label, + "similarity": similarity, + "correct": label in expected_logos, + }) + + elif args.matching_method == "margin": # Margin-based matching: requires margin over second-best match_result = detector.find_best_match_with_margin( detection["embedding"], @@ -444,7 +465,20 @@ def main(): ) if match_result: label, similarity = match_result - match = label + matched_logos.add(label) + + if label in expected_logos: + true_positives += 1 + else: + false_positives += 1 + + results.append({ + "test_image": test_filename, + "matched_logo": label, + "similarity": similarity, + "correct": label in expected_logos, + }) + else: # multi-ref # Multi-ref matching: aggregates scores across reference images match_result = detector.find_best_match_multi_ref( @@ -457,23 +491,19 @@ def main(): ) if match_result: label, similarity, num_matching = match_result - match = label + matched_logos.add(label) - if match: - matched_logos.add(match) + if label in expected_logos: + true_positives += 1 + else: + false_positives += 1 - # Check if this is a correct match - if match in expected_logos: - true_positives += 1 - else: - false_positives += 1 - - results.append({ - "test_image": test_filename, - "matched_logo": match, - "similarity": similarity, - "correct": match in expected_logos, - }) + results.append({ + "test_image": test_filename, + "matched_logo": label, + "similarity": similarity, + "correct": label in expected_logos, + }) # Count missed detections (false negatives) missed = expected_logos - matched_logos @@ -512,7 +542,8 @@ def main(): print(f" CLIP similarity threshold: {args.threshold}") print(f" DETR confidence threshold: {args.detr_threshold}") print(f" Matching method: {args.matching_method}") - print(f" Matching margin: {args.margin}") + if args.matching_method in ("margin", "multi-ref"): + print(f" Matching margin: {args.margin}") if args.matching_method == "multi-ref": print(f" Min matching refs: {args.min_matching_refs}") print(f" Similarity aggregation: {'max' if args.use_max_similarity else 'mean'}")