Add simple matching method as baseline for comparison tests
- Add find_all_matches() method to DetectLogosDETR that returns all logos above similarity threshold without any rejection logic - Add --matching-method simple option to test script - Update run_comparison_tests.sh to include simple matching as Test 1 - Update documentation to describe simple matching method
This commit is contained in:
@ -394,6 +394,47 @@ class DetectLogosDETR:
|
|||||||
else:
|
else:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def find_all_matches(
|
||||||
|
self,
|
||||||
|
detected_embedding: torch.Tensor,
|
||||||
|
reference_embeddings: List[Tuple[str, torch.Tensor]],
|
||||||
|
similarity_threshold: float = 0.7,
|
||||||
|
) -> List[Tuple[str, float]]:
|
||||||
|
"""
|
||||||
|
Find all matching reference logos above the similarity threshold.
|
||||||
|
|
||||||
|
Unlike find_best_match, this returns ALL logos that have at least one
|
||||||
|
reference above threshold. Each unique logo is returned once with its
|
||||||
|
highest similarity score.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
detected_embedding: CLIP embedding from detected logo region
|
||||||
|
reference_embeddings: List of (label, embedding) tuples for reference logos
|
||||||
|
similarity_threshold: Minimum similarity to consider a match (0-1)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of (label, similarity) tuples for all matches above threshold,
|
||||||
|
sorted by similarity descending. Each logo appears at most once.
|
||||||
|
"""
|
||||||
|
if not reference_embeddings:
|
||||||
|
return []
|
||||||
|
|
||||||
|
# Track best similarity for each logo
|
||||||
|
logo_best_sim: Dict[str, float] = {}
|
||||||
|
|
||||||
|
for label, ref_embedding in reference_embeddings:
|
||||||
|
similarity = self.compare_embeddings(detected_embedding, ref_embedding)
|
||||||
|
|
||||||
|
if similarity >= similarity_threshold:
|
||||||
|
if label not in logo_best_sim or similarity > logo_best_sim[label]:
|
||||||
|
logo_best_sim[label] = similarity
|
||||||
|
|
||||||
|
# Convert to list and sort by similarity descending
|
||||||
|
matches = [(label, sim) for label, sim in logo_best_sim.items()]
|
||||||
|
matches.sort(key=lambda x: x[1], reverse=True)
|
||||||
|
|
||||||
|
return matches
|
||||||
|
|
||||||
def find_best_match_multi_ref(
|
def find_best_match_multi_ref(
|
||||||
self,
|
self,
|
||||||
detected_embedding: torch.Tensor,
|
detected_embedding: torch.Tensor,
|
||||||
|
|||||||
@ -78,6 +78,41 @@ match = detector.find_best_match(
|
|||||||
**Returns:**
|
**Returns:**
|
||||||
- Tuple of (label, similarity) for best match, or None if no match above threshold
|
- Tuple of (label, similarity) for best match, or None if no match above threshold
|
||||||
|
|
||||||
|
#### `find_all_matches()` - Find all matching reference logos
|
||||||
|
|
||||||
|
Returns ALL logos that have at least one reference above the similarity threshold. Each unique logo appears once with its highest similarity score.
|
||||||
|
|
||||||
|
```python
|
||||||
|
matches = detector.find_all_matches(
|
||||||
|
detected_embedding,
|
||||||
|
reference_embeddings,
|
||||||
|
similarity_threshold=0.7
|
||||||
|
)
|
||||||
|
# Returns: [(label1, similarity1), (label2, similarity2), ...]
|
||||||
|
```
|
||||||
|
|
||||||
|
**Parameters:**
|
||||||
|
- `detected_embedding`: CLIP embedding from detected logo region
|
||||||
|
- `reference_embeddings`: List of (label, embedding) tuples for reference logos
|
||||||
|
- `similarity_threshold`: Minimum similarity to consider a match (0-1, default: 0.7)
|
||||||
|
|
||||||
|
**Returns:**
|
||||||
|
- List of (label, similarity) tuples for all matches above threshold, sorted by similarity descending
|
||||||
|
- Each logo appears at most once (with its highest matching reference)
|
||||||
|
|
||||||
|
**Example:**
|
||||||
|
```python
|
||||||
|
# Get all logos that match a detection
|
||||||
|
all_matches = detector.find_all_matches(
|
||||||
|
detection["embedding"],
|
||||||
|
reference_embeddings,
|
||||||
|
similarity_threshold=0.7
|
||||||
|
)
|
||||||
|
|
||||||
|
for label, similarity in all_matches:
|
||||||
|
print(f"Matched: {label} (similarity: {similarity:.3f})")
|
||||||
|
```
|
||||||
|
|
||||||
#### `detect_and_match()` - One-step detection and matching
|
#### `detect_and_match()` - One-step detection and matching
|
||||||
|
|
||||||
```python
|
```python
|
||||||
|
|||||||
@ -39,8 +39,8 @@ The system uses a two-stage pipeline:
|
|||||||
|
|
||||||
| Parameter | Default | Description |
|
| Parameter | Default | Description |
|
||||||
|-----------|---------|-------------|
|
|-----------|---------|-------------|
|
||||||
| `--matching-method` | margin | Matching method: `margin` or `multi-ref` |
|
| `--matching-method` | margin | Matching method: `simple`, `margin`, or `multi-ref` |
|
||||||
| `--margin` | 0.05 | Required margin between best and second-best match (applies to both methods) |
|
| `--margin` | 0.05 | Required margin between best and second-best match (applies to `margin` and `multi-ref`) |
|
||||||
|
|
||||||
#### Multi-Ref Method Parameters (when `--matching-method multi-ref`)
|
#### Multi-Ref Method Parameters (when `--matching-method multi-ref`)
|
||||||
|
|
||||||
@ -193,11 +193,11 @@ This ensures cosine similarity is computed correctly and scores fall in the rang
|
|||||||
|
|
||||||
| Method | Test Script Option | Key Feature |
|
| Method | Test Script Option | Key Feature |
|
||||||
|--------|-------------------|-------------|
|
|--------|-------------------|-------------|
|
||||||
| `find_best_match` | N/A (library only) | Returns highest similarity above threshold |
|
| `find_all_matches` | `--matching-method simple` | Returns ALL logos above threshold (baseline, most permissive) |
|
||||||
| `find_best_match_with_margin` | `--matching-method margin` | Requires margin over second-best match |
|
| `find_best_match_with_margin` | `--matching-method margin` | Requires margin over second-best match |
|
||||||
| `find_best_match_multi_ref` | `--matching-method multi-ref` | Aggregates scores across reference images |
|
| `find_best_match_multi_ref` | `--matching-method multi-ref` | Aggregates scores across reference images |
|
||||||
|
|
||||||
The test script supports both `margin` and `multi-ref` matching methods via the `--matching-method` parameter.
|
The test script supports `simple`, `margin`, and `multi-ref` matching methods via the `--matching-method` parameter.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@ -242,13 +242,14 @@ Input Image
|
|||||||
▼
|
▼
|
||||||
┌─────────────────────────────────────┐
|
┌─────────────────────────────────────┐
|
||||||
│ Matching (selectable method) │
|
│ Matching (selectable method) │
|
||||||
│ ┌───────────────┬────────────────┐ │
|
│ ┌─────────┬─────────┬────────────┐ │
|
||||||
│ │ margin │ multi-ref │ │
|
│ │ simple │ margin │ multi-ref │ │
|
||||||
│ ├───────────────┼────────────────┤ │
|
│ ├─────────┼─────────┼────────────┤ │
|
||||||
│ │ Require margin│ Aggregate │ │
|
│ │ All │ Require │ Aggregate │ │
|
||||||
│ │ over 2nd best │ across refs │ │
|
│ │ matches │ margin │ across │ │
|
||||||
│ │ match │ (mean or max) │ │
|
│ │ above │ over │ refs │ │
|
||||||
│ └───────────────┴────────────────┘ │
|
│ │ thresh │ 2nd best│ (mean/max) │ │
|
||||||
|
│ └─────────┴─────────┴────────────┘ │
|
||||||
└─────────────────────────────────────┘
|
└─────────────────────────────────────┘
|
||||||
│
|
│
|
||||||
▼
|
▼
|
||||||
@ -259,6 +260,15 @@ Matched Logo Labels
|
|||||||
|
|
||||||
## Tuning Recommendations
|
## Tuning Recommendations
|
||||||
|
|
||||||
|
### For Simple Matching (`--matching-method simple`)
|
||||||
|
|
||||||
|
| Goal | Adjustments |
|
||||||
|
|------|-------------|
|
||||||
|
| **Reduce false positives** | Increase `--threshold` (only tuning option for simple method) |
|
||||||
|
| **Reduce false negatives** | Decrease `--threshold` |
|
||||||
|
|
||||||
|
Note: Simple matching is primarily used as a baseline. For production use, consider `margin` or `multi-ref`.
|
||||||
|
|
||||||
### For Margin-Based Matching (`--matching-method margin`)
|
### For Margin-Based Matching (`--matching-method margin`)
|
||||||
|
|
||||||
| Goal | Adjustments |
|
| Goal | Adjustments |
|
||||||
@ -287,6 +297,9 @@ Matched Logo Labels
|
|||||||
## Example Usage
|
## Example Usage
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
# Simple matching (baseline - all matches above threshold)
|
||||||
|
python test_logo_detection.py -n 20 --matching-method simple --threshold 0.70
|
||||||
|
|
||||||
# Default margin-based matching
|
# Default margin-based matching
|
||||||
python test_logo_detection.py -n 20 --threshold 0.75 --margin 0.05
|
python test_logo_detection.py -n 20 --threshold 0.75 --margin 0.05
|
||||||
|
|
||||||
|
|||||||
@ -1,6 +1,6 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
#
|
#
|
||||||
# Run logo detection tests with all three matching methods and save results.
|
# Run logo detection tests with all four matching methods and save results.
|
||||||
#
|
#
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
@ -30,8 +30,22 @@ echo " Min matching refs: $MIN_MATCHING_REFS"
|
|||||||
echo " Seed: $SEED"
|
echo " Seed: $SEED"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
# Test 1: Margin-based matching
|
# Test 1: Simple matching (baseline - all matches above threshold)
|
||||||
echo "=== Test 1: Margin-based matching ===" | tee -a "$OUTPUT_FILE"
|
echo "=== Test 1: Simple matching (baseline) ===" | tee -a "$OUTPUT_FILE"
|
||||||
|
uv run python "$SCRIPT_DIR/test_logo_detection.py" \
|
||||||
|
--num-logos $NUM_LOGOS \
|
||||||
|
--refs-per-logo $REFS_PER_LOGO \
|
||||||
|
--positive-samples $POSITIVE_SAMPLES \
|
||||||
|
--negative-samples $NEGATIVE_SAMPLES \
|
||||||
|
--matching-method simple \
|
||||||
|
--seed $SEED \
|
||||||
|
2>&1 | tee -a "$OUTPUT_FILE"
|
||||||
|
|
||||||
|
echo "" >> "$OUTPUT_FILE"
|
||||||
|
echo "" >> "$OUTPUT_FILE"
|
||||||
|
|
||||||
|
# Test 2: Margin-based matching
|
||||||
|
echo "=== Test 2: Margin-based matching ===" | tee -a "$OUTPUT_FILE"
|
||||||
uv run python "$SCRIPT_DIR/test_logo_detection.py" \
|
uv run python "$SCRIPT_DIR/test_logo_detection.py" \
|
||||||
--num-logos $NUM_LOGOS \
|
--num-logos $NUM_LOGOS \
|
||||||
--refs-per-logo $REFS_PER_LOGO \
|
--refs-per-logo $REFS_PER_LOGO \
|
||||||
@ -44,8 +58,8 @@ uv run python "$SCRIPT_DIR/test_logo_detection.py" \
|
|||||||
echo "" >> "$OUTPUT_FILE"
|
echo "" >> "$OUTPUT_FILE"
|
||||||
echo "" >> "$OUTPUT_FILE"
|
echo "" >> "$OUTPUT_FILE"
|
||||||
|
|
||||||
# Test 2: Multi-ref with mean similarity
|
# Test 3: Multi-ref with mean similarity
|
||||||
echo "=== Test 2: Multi-ref matching (mean similarity) ===" | tee -a "$OUTPUT_FILE"
|
echo "=== Test 3: Multi-ref matching (mean similarity) ===" | tee -a "$OUTPUT_FILE"
|
||||||
uv run python "$SCRIPT_DIR/test_logo_detection.py" \
|
uv run python "$SCRIPT_DIR/test_logo_detection.py" \
|
||||||
--num-logos $NUM_LOGOS \
|
--num-logos $NUM_LOGOS \
|
||||||
--refs-per-logo $REFS_PER_LOGO \
|
--refs-per-logo $REFS_PER_LOGO \
|
||||||
@ -59,8 +73,8 @@ uv run python "$SCRIPT_DIR/test_logo_detection.py" \
|
|||||||
echo "" >> "$OUTPUT_FILE"
|
echo "" >> "$OUTPUT_FILE"
|
||||||
echo "" >> "$OUTPUT_FILE"
|
echo "" >> "$OUTPUT_FILE"
|
||||||
|
|
||||||
# Test 3: Multi-ref with max similarity
|
# Test 4: Multi-ref with max similarity
|
||||||
echo "=== Test 3: Multi-ref matching (max similarity) ===" | tee -a "$OUTPUT_FILE"
|
echo "=== Test 4: Multi-ref matching (max similarity) ===" | tee -a "$OUTPUT_FILE"
|
||||||
uv run python "$SCRIPT_DIR/test_logo_detection.py" \
|
uv run python "$SCRIPT_DIR/test_logo_detection.py" \
|
||||||
--num-logos $NUM_LOGOS \
|
--num-logos $NUM_LOGOS \
|
||||||
--refs-per-logo $REFS_PER_LOGO \
|
--refs-per-logo $REFS_PER_LOGO \
|
||||||
|
|||||||
@ -236,9 +236,10 @@ def main():
|
|||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--matching-method",
|
"--matching-method",
|
||||||
type=str,
|
type=str,
|
||||||
choices=["margin", "multi-ref"],
|
choices=["simple", "margin", "multi-ref"],
|
||||||
default="margin",
|
default="margin",
|
||||||
help="Matching method: 'margin' requires confidence margin over 2nd best, "
|
help="Matching method: 'simple' returns all matches above threshold, "
|
||||||
|
"'margin' requires confidence margin over 2nd best, "
|
||||||
"'multi-ref' aggregates scores across reference images (default: margin)",
|
"'multi-ref' aggregates scores across reference images (default: margin)",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
@ -431,10 +432,30 @@ def main():
|
|||||||
# Match detections against references using selected method
|
# Match detections against references using selected method
|
||||||
matched_logos: Set[str] = set()
|
matched_logos: Set[str] = set()
|
||||||
for detection in detections:
|
for detection in detections:
|
||||||
match = None
|
if args.matching_method == "simple":
|
||||||
similarity = None
|
# Simple matching: return ALL logos above threshold
|
||||||
|
all_matches = detector.find_all_matches(
|
||||||
|
detection["embedding"],
|
||||||
|
reference_embeddings,
|
||||||
|
similarity_threshold=args.threshold,
|
||||||
|
)
|
||||||
|
for label, similarity in all_matches:
|
||||||
|
matched_logos.add(label)
|
||||||
|
|
||||||
if args.matching_method == "margin":
|
# Check if this is a correct match
|
||||||
|
if label in expected_logos:
|
||||||
|
true_positives += 1
|
||||||
|
else:
|
||||||
|
false_positives += 1
|
||||||
|
|
||||||
|
results.append({
|
||||||
|
"test_image": test_filename,
|
||||||
|
"matched_logo": label,
|
||||||
|
"similarity": similarity,
|
||||||
|
"correct": label in expected_logos,
|
||||||
|
})
|
||||||
|
|
||||||
|
elif args.matching_method == "margin":
|
||||||
# Margin-based matching: requires margin over second-best
|
# Margin-based matching: requires margin over second-best
|
||||||
match_result = detector.find_best_match_with_margin(
|
match_result = detector.find_best_match_with_margin(
|
||||||
detection["embedding"],
|
detection["embedding"],
|
||||||
@ -444,7 +465,20 @@ def main():
|
|||||||
)
|
)
|
||||||
if match_result:
|
if match_result:
|
||||||
label, similarity = match_result
|
label, similarity = match_result
|
||||||
match = label
|
matched_logos.add(label)
|
||||||
|
|
||||||
|
if label in expected_logos:
|
||||||
|
true_positives += 1
|
||||||
|
else:
|
||||||
|
false_positives += 1
|
||||||
|
|
||||||
|
results.append({
|
||||||
|
"test_image": test_filename,
|
||||||
|
"matched_logo": label,
|
||||||
|
"similarity": similarity,
|
||||||
|
"correct": label in expected_logos,
|
||||||
|
})
|
||||||
|
|
||||||
else: # multi-ref
|
else: # multi-ref
|
||||||
# Multi-ref matching: aggregates scores across reference images
|
# Multi-ref matching: aggregates scores across reference images
|
||||||
match_result = detector.find_best_match_multi_ref(
|
match_result = detector.find_best_match_multi_ref(
|
||||||
@ -457,23 +491,19 @@ def main():
|
|||||||
)
|
)
|
||||||
if match_result:
|
if match_result:
|
||||||
label, similarity, num_matching = match_result
|
label, similarity, num_matching = match_result
|
||||||
match = label
|
matched_logos.add(label)
|
||||||
|
|
||||||
if match:
|
if label in expected_logos:
|
||||||
matched_logos.add(match)
|
true_positives += 1
|
||||||
|
else:
|
||||||
|
false_positives += 1
|
||||||
|
|
||||||
# Check if this is a correct match
|
results.append({
|
||||||
if match in expected_logos:
|
"test_image": test_filename,
|
||||||
true_positives += 1
|
"matched_logo": label,
|
||||||
else:
|
"similarity": similarity,
|
||||||
false_positives += 1
|
"correct": label in expected_logos,
|
||||||
|
})
|
||||||
results.append({
|
|
||||||
"test_image": test_filename,
|
|
||||||
"matched_logo": match,
|
|
||||||
"similarity": similarity,
|
|
||||||
"correct": match in expected_logos,
|
|
||||||
})
|
|
||||||
|
|
||||||
# Count missed detections (false negatives)
|
# Count missed detections (false negatives)
|
||||||
missed = expected_logos - matched_logos
|
missed = expected_logos - matched_logos
|
||||||
@ -512,7 +542,8 @@ def main():
|
|||||||
print(f" CLIP similarity threshold: {args.threshold}")
|
print(f" CLIP similarity threshold: {args.threshold}")
|
||||||
print(f" DETR confidence threshold: {args.detr_threshold}")
|
print(f" DETR confidence threshold: {args.detr_threshold}")
|
||||||
print(f" Matching method: {args.matching_method}")
|
print(f" Matching method: {args.matching_method}")
|
||||||
print(f" Matching margin: {args.margin}")
|
if args.matching_method in ("margin", "multi-ref"):
|
||||||
|
print(f" Matching margin: {args.margin}")
|
||||||
if args.matching_method == "multi-ref":
|
if args.matching_method == "multi-ref":
|
||||||
print(f" Min matching refs: {args.min_matching_refs}")
|
print(f" Min matching refs: {args.min_matching_refs}")
|
||||||
print(f" Similarity aggregation: {'max' if args.use_max_similarity else 'mean'}")
|
print(f" Similarity aggregation: {'max' if args.use_max_similarity else 'mean'}")
|
||||||
|
|||||||
Reference in New Issue
Block a user