可视化操作界面加入开放集测试。

exp_multimodal/exp_multimodal_gui.py

@@ -13,9 +13,12 @@ from tkinter import filedialog, messagebox
 from tkinterdnd2 import DND_FILES, TkinterDnD
 
 from exp_multimodal.labels import build_labels, _normalize, _base_ingredient
-from exp_multimodal.vlm_classifier import classify_image
+from exp_multimodal.vlm_classifier import classify_image, classify_image_openset
 from exp_multimodal.ollama_client import OLLAMA_URL as DEFAULT_OLLAMA_URL, DEFAULT_MODEL as DEFAULT_VLM_MODEL
 from exp_multimodal.vlm_providers import VLMProvider, OllamaProvider, KimiProvider
+from exp_multimodal.text_embedder import OllamaEmbedder
+from exp_multimodal.vector_matcher import DishNameMatcher
+from exp_multimodal.build_dish_name_index import build_index as build_dish_index
 
 
 ctk.set_appearance_mode("System")
@@ -63,6 +66,17 @@ class MultiModalFoodApp:
         }
         self.extra_labels_file_path: Optional[str] = None
 
+        # 开放式识别配置
+        self.openset_enabled_var = ctk.BooleanVar(value=False)
+        self.openset_index_path_var = ctk.StringVar(value="exp_multimodal/dish_name_index")
+        self.openset_top_k_var = ctk.IntVar(value=3)
+        self.openset_min_score_var = ctk.DoubleVar(value=0.5)
+        self.openset_embedder_model_var = ctk.StringVar(value="quentinz/bge-large-zh-v1.5")
+        
+        # 开放式识别运行时对象（延迟初始化）
+        self.openset_embedder: Optional[OllamaEmbedder] = None
+        self.openset_matcher: Optional[DishNameMatcher] = None
+
         # UI
         self.create_widgets()
         
@@ -309,7 +323,7 @@ class MultiModalFoodApp:
             info.pack(side="left", fill="both", expand=True, padx=10, pady=10)
             ctk.CTkLabel(info, text=f"文件: {r['image_name']}", anchor="w", font=("Arial", 12, "bold")).pack(fill="x", padx=5, pady=2)
 
-            # 仅展示解析后的标签和置信度
+            # 显示预测结果
             is_correct = r.get('is_correct')
             color = "green" if is_correct is True else ("red" if is_correct is False else "orange")
             ctk.CTkLabel(info, text=r['predicted_label'], anchor="w", font=("Arial", 18, "bold"), text_color=color).pack(fill="x", padx=5, pady=(4, 2))
@@ -319,6 +333,37 @@ class MultiModalFoodApp:
             true_cls = r.get('true_class')
             if true_cls:
                 ctk.CTkLabel(info, text=f"真实类别: {true_cls}", anchor="w", font=("Arial", 10)).pack(fill="x", padx=5, pady=(0, 2))
+            
+            # 开放式识别额外信息
+            if r.get('openset_mode'):
+                raw_dish = r.get('raw_dish', '')
+                if raw_dish:
+                    ctk.CTkLabel(info, text=f"VLM原始输出: {raw_dish}", anchor="w", font=("Arial", 9), text_color="gray").pack(fill="x", padx=5, pady=(0, 2))
+                
+                # 候选列表（可展开）
+                candidates = r.get('candidates', [])
+                if candidates:
+                    # 创建可展开的候选框
+                    candidates_frame = ctk.CTkFrame(info)
+                    candidates_frame.pack(fill="x", padx=5, pady=(4, 2))
+                    
+                    def toggle_candidates(frame=candidates_frame, cands=candidates):
+                        # 切换显示/隐藏
+                        if len(frame.winfo_children()) > 1:
+                            # 隐藏
+                            for w in frame.winfo_children()[1:]:
+                                w.destroy()
+                        else:
+                            # 显示
+                            for idx, cand in enumerate(cands, 1):
+                                dish_name = cand.get('dish', 'Unknown')
+                                match_score = cand.get('match_score', 0.0)
+                                final_score = cand.get('final_score', 0.0)
+                                cand_text = f"  [{idx}] {dish_name} (匹配:{match_score:.3f} 综合:{final_score:.3f})"
+                                ctk.CTkLabel(frame, text=cand_text, anchor="w", font=("Arial", 9), text_color="gray").pack(fill="x", padx=5, pady=1)
+                    
+                    toggle_btn = ctk.CTkButton(candidates_frame, text=f"▶ 查看候选列表 ({len(candidates)}个)", command=toggle_candidates, width=180, height=24, fg_color="gray", hover_color="darkgray")
+                    toggle_btn.pack(anchor="w", padx=5, pady=2)
 
     # -------------------- 右侧-配置与运行 --------------------
     def create_config_tab(self):
@@ -439,6 +484,44 @@ class MultiModalFoodApp:
         self.refresh_extra_labels_view()
         self.refresh_fewshot_view()
 
+        # ==================== 开放式识别配置 ====================
+        openset_frame = ctk.CTkFrame(panel)
+        openset_frame.pack(fill="x", pady=(10, 10))
+        
+        top = ctk.CTkFrame(openset_frame)
+        top.pack(fill="x")
+        ctk.CTkLabel(top, text="开放式识别配置", font=("Arial", 14, "bold")).pack(side="left", padx=0, pady=(0, 6))
+        ctk.CTkSwitch(top, text="启用（仅dish模式）", variable=self.openset_enabled_var, command=self.on_openset_toggle).pack(side="left", padx=12)
+        
+        # 索引路径
+        openset_row1 = ctk.CTkFrame(openset_frame)
+        openset_row1.pack(fill="x", pady=(4, 4))
+        ctk.CTkLabel(openset_row1, text="向量索引路径:").pack(side="left", padx=6, pady=4)
+        ctk.CTkEntry(openset_row1, textvariable=self.openset_index_path_var, width=320).pack(side="left", padx=4, pady=4)
+        ctk.CTkButton(openset_row1, text="浏览", command=self.browse_openset_index, width=80).pack(side="left", padx=4)
+        
+        # Top-K 和 最低分数
+        openset_row2 = ctk.CTkFrame(openset_frame)
+        openset_row2.pack(fill="x", pady=(0, 4))
+        ctk.CTkLabel(openset_row2, text="Top-K候选数:").pack(side="left", padx=6, pady=4)
+        ctk.CTkEntry(openset_row2, textvariable=self.openset_top_k_var, width=80).pack(side="left", padx=4, pady=4)
+        ctk.CTkLabel(openset_row2, text="最低匹配分数:").pack(side="left", padx=(20, 6), pady=4)
+        ctk.CTkEntry(openset_row2, textvariable=self.openset_min_score_var, width=80).pack(side="left", padx=4, pady=4)
+        
+        # Embedding 模型
+        openset_row3 = ctk.CTkFrame(openset_frame)
+        openset_row3.pack(fill="x", pady=(0, 4))
+        ctk.CTkLabel(openset_row3, text="Embedding模型:").pack(side="left", padx=6, pady=4)
+        ctk.CTkEntry(openset_row3, textvariable=self.openset_embedder_model_var, width=380).pack(side="left", padx=4, pady=4)
+        
+        # 索引管理按钮
+        openset_row4 = ctk.CTkFrame(openset_frame)
+        openset_row4.pack(fill="x", pady=(4, 6))
+        ctk.CTkButton(openset_row4, text="构建索引", command=self.build_openset_index, width=110, fg_color="green", hover_color="darkgreen").pack(side="left", padx=6)
+        ctk.CTkButton(openset_row4, text="测试索引", command=self.test_openset_index, width=110).pack(side="left", padx=6)
+        self.openset_status_label = ctk.CTkLabel(openset_row4, text="", font=("Arial", 10), text_color="gray")
+        self.openset_status_label.pack(side="left", padx=12)
+
         # 配置保存/加载
         cfg_manage = ctk.CTkFrame(panel)
         cfg_manage.pack(fill="x", pady=(10, 10))
@@ -602,6 +685,96 @@ class MultiModalFoodApp:
         except Exception as e:
             messagebox.showerror("错误", f"保存失败: {e}")
 
+    # -------------------- 开放式识别配置管理 --------------------
+    def on_openset_toggle(self):
+        """开放式识别开关切换时的处理"""
+        if self.openset_enabled_var.get():
+            # 提示用户该功能仅适用于 dish 模式
+            if self.mode_var.get() != "dish":
+                messagebox.showwarning("提示", "开放式识别当前仅支持 dish 模式，请先切换模式")
+                self.openset_enabled_var.set(False)
+                return
+            
+            # 检查索引是否存在
+            index_path = self.openset_index_path_var.get()
+            if not os.path.exists(index_path):
+                if messagebox.askyesno("索引不存在", f"索引目录 {index_path} 不存在。是否现在构建索引？"):
+                    self.build_openset_index()
+                else:
+                    self.openset_enabled_var.set(False)
+    
+    def browse_openset_index(self):
+        """浏览选择索引目录"""
+        path = filedialog.askdirectory(title="选择向量索引目录")
+        if path:
+            self.openset_index_path_var.set(path)
+    
+    def build_openset_index(self):
+        """构建向量索引（后台线程）"""
+        if messagebox.askyesno("确认", "构建索引可能需要几分钟时间，是否继续？"):
+            self.openset_status_label.configure(text="正在构建索引...", text_color="orange")
+            threading.Thread(target=self._build_index_thread, daemon=True).start()
+    
+    def _build_index_thread(self):
+        """索引构建后台线程"""
+        try:
+            mode = "dish"  # 开放式识别目前仅支持 dish
+            labels = build_labels(mode, alias_map_path=None)
+            
+            if not labels:
+                self.root.after(0, lambda: messagebox.showerror("错误", "未找到可用的菜品名"))
+                self.root.after(0, lambda: self.openset_status_label.configure(text="构建失败", text_color="red"))
+                return
+            
+            # 初始化 Embedder
+            embedder_url = self.ollama_url_var.get().strip() or DEFAULT_OLLAMA_URL
+            embedder_model = self.openset_embedder_model_var.get().strip()
+            embedder = OllamaEmbedder(base_url=embedder_url, model=embedder_model)
+            
+            # 构建索引
+            output_dir = self.openset_index_path_var.get()
+            build_dish_index(
+                dish_names=labels,
+                embedder=embedder,
+                output_dir=output_dir,
+                batch_size=100
+            )
+            
+            self.root.after(0, lambda: messagebox.showinfo("成功", f"索引构建完成！位置: {output_dir}"))
+            self.root.after(0, lambda: self.openset_status_label.configure(text=f"索引已构建 ({len(labels)}个菜品)", text_color="green"))
+            
+        except Exception as e:
+            self.root.after(0, lambda: messagebox.showerror("错误", f"构建索引失败: {e}"))
+            self.root.after(0, lambda: self.openset_status_label.configure(text="构建失败", text_color="red"))
+    
+    def test_openset_index(self):
+        """测试索引是否可用"""
+        try:
+            index_path = self.openset_index_path_var.get()
+            
+            # 检查必要文件是否存在
+            required_files = ["dish_names.json", "dish_embeddings.npy", "faiss_index.bin"]
+            missing = []
+            for fname in required_files:
+                if not os.path.exists(os.path.join(index_path, fname)):
+                    missing.append(fname)
+            
+            if missing:
+                messagebox.showerror("错误", f"索引目录不完整，缺少文件:{', '.join(missing)}")
+                self.openset_status_label.configure(text="索引无效", text_color="red")
+                return
+            
+            # 尝试加载索引
+            matcher = DishNameMatcher(index_dir=index_path)
+            num_dishes = len(matcher.dish_names)
+            
+            messagebox.showinfo("成功", f"索引测试通过！包含 {num_dishes} 个菜品名")
+            self.openset_status_label.configure(text=f"索引正常 ({num_dishes}个菜品)", text_color="green")
+            
+        except Exception as e:
+            messagebox.showerror("错误", f"索引测试失败: {e}")
+            self.openset_status_label.configure(text="索引无效", text_color="red")
+
     # -------------------- 识别流程 --------------------
     def start_recognition(self):
         if not self.uploaded_images:
@@ -633,50 +806,138 @@ class MultiModalFoodApp:
     def recognize_images(self):
         try:
             mode = self.mode_var.get()
-            labels = self._build_final_labels(mode)
+            use_openset = self.openset_enabled_var.get() and mode == "dish"
-            ingredient_only = (mode in {"whole", "processed"})
+            
-            use_hints = self.fewshot_hints if self.fewshot_enabled_var.get() else None
-
             # 创建 VLM Provider
             provider = self._create_vlm_provider()
             if provider is None:
                 self.root.after(0, lambda: messagebox.showerror("错误", "Provider 配置错误，请检查配置"))
                 self.root.after(0, self.recognition_completed)
                 return
+            
+            # 开放式识别分支
+            if use_openset:
+                self._recognize_with_openset(provider)
+            # 封闭式识别分支（原有逻辑）
+            else:
+                self._recognize_with_closedset(provider, mode)
+            
+            self.root.after(0, self.recognition_completed)
+        except Exception as e:
+            self.root.after(0, lambda: messagebox.showerror("错误", f"识别过程中出错: {e}"))
+            self.root.after(0, self.recognition_completed)
+    
+    def _recognize_with_closedset(self, provider: VLMProvider, mode: str):
+        """封闭式识别（原有逻辑）"""
+        labels = self._build_final_labels(mode)
+        ingredient_only = (mode in {"whole", "processed"})
+        use_hints = self.fewshot_hints if self.fewshot_enabled_var.get() else None
 
+        self.current_results.clear()
+        for i, img in enumerate(self.uploaded_images):
+            try:
+                result = classify_image(
+                    image_path=img['path'],
+                    labels=labels,
+                    provider=provider,
+                    fewshot_hints=use_hints,
+                    ingredient_only=ingredient_only,
+                )
+                pred = result.get("label", "Unknown")
+                conf = result.get("confidence", 0.0)
+                # 若非数值,后续展示为 N/A
+                if not isinstance(conf, (int, float)):
+                    try:
+                        conf = float(conf)
+                    except Exception:
+                        pass
+                true_cls = self._infer_true_class(img['path'], labels)
+                is_correct = (pred == true_cls) if true_cls is not None else None
+                ui_res = {
+                    "image_index": i,
+                    "image_name": img['name'],
+                    "predicted_label": pred,
+                    "confidence": conf,
+                    "true_class": true_cls,
+                    "is_correct": is_correct,
+                    "openset_mode": False,
+                }
+                self.current_results.append(ui_res)
+                img['recognized'] = True
+                img['result'] = ui_res
+            except Exception as e:
+                print(f"[VLM] classify error: {e}")
+                ui_res = {
+                    "image_index": i,
+                    "image_name": img['name'],
+                    "predicted_label": "Error",
+                    "confidence": "N/A",
+                    "true_class": None,
+                    "is_correct": None,
+                    "openset_mode": False,
+                }
+                self.current_results.append(ui_res)
+                img['recognized'] = True
+                img['result'] = ui_res
+            finally:
+                self.root.after(0, self.update_progress, i + 1, len(self.uploaded_images))
+    
+    def _recognize_with_openset(self, provider: VLMProvider):
+        """开放式识别（新逻辑）"""
+        try:
+            # 初始化 Embedder 和 Matcher（延迟加载）
+            if self.openset_embedder is None:
+                embedder_url = self.ollama_url_var.get().strip() or DEFAULT_OLLAMA_URL
+                embedder_model = self.openset_embedder_model_var.get().strip()
+                self.openset_embedder = OllamaEmbedder(base_url=embedder_url, model=embedder_model)
+            
+            if self.openset_matcher is None:
+                index_path = self.openset_index_path_var.get()
+                self.openset_matcher = DishNameMatcher(index_dir=index_path)
+            
+            top_k = self.openset_top_k_var.get()
+            min_score = self.openset_min_score_var.get()
+            
+            # 获取所有菜品名（用于准确率计算）
+            all_dish_names = self.openset_matcher.dish_names
+            
             self.current_results.clear()
             for i, img in enumerate(self.uploaded_images):
                 try:
-                    result = classify_image(
+                    result = classify_image_openset(
                         image_path=img['path'],
-                        labels=labels,
                         provider=provider,
-                        fewshot_hints=use_hints,
+                        embedder=self.openset_embedder,
-                        ingredient_only=ingredient_only,
+                        matcher=self.openset_matcher,
+                        top_k=top_k,
+                        min_match_score=min_score,
                     )
-                    pred = result.get("label", "Unknown")
+                    
-                    conf = result.get("confidence", 0.0)
+                    # 解析开放式结果
-                    # 若非数值,后续展示为 N/A
+                    best_match = result.get("best_match", "Unknown")
-                    if not isinstance(conf, (int, float)):
+                    final_conf = result.get("final_confidence", 0.0)
-                        try:
+                    candidates = result.get("candidates", [])
-                            conf = float(conf)
+                    raw_dish = result.get("raw_dish", "")
-                        except Exception:
+                    
-                            pass
+                    true_cls = self._infer_true_class(img['path'], all_dish_names)
-                    true_cls = self._infer_true_class(img['path'], labels)
+                    is_correct = (best_match == true_cls) if true_cls is not None else None
-                    is_correct = (pred == true_cls) if true_cls is not None else None
+                    
                     ui_res = {
                         "image_index": i,
                         "image_name": img['name'],
-                        "predicted_label": pred,
+                        "predicted_label": best_match,
-                        "confidence": conf,
+                        "confidence": final_conf,
                         "true_class": true_cls,
                         "is_correct": is_correct,
+                        "openset_mode": True,
+                        "raw_dish": raw_dish,
+                        "candidates": candidates,
                     }
                     self.current_results.append(ui_res)
                     img['recognized'] = True
                     img['result'] = ui_res
                 except Exception as e:
-                    print(f"[VLM] classify error: {e}")
+                    print(f"[VLM-Openset] classify error: {e}")
                     ui_res = {
                         "image_index": i,
                         "image_name": img['name'],
@@ -684,16 +945,18 @@ class MultiModalFoodApp:
                         "confidence": "N/A",
                         "true_class": None,
                         "is_correct": None,
+                        "openset_mode": True,
+                        "raw_dish": "",
+                        "candidates": [],
                     }
                     self.current_results.append(ui_res)
                     img['recognized'] = True
                     img['result'] = ui_res
                 finally:
                     self.root.after(0, self.update_progress, i + 1, len(self.uploaded_images))
-            self.root.after(0, self.recognition_completed)
+        
         except Exception as e:
-            self.root.after(0, lambda: messagebox.showerror("错误", f"识别过程中出错: {e}"))
+            raise Exception(f"开放式识别初始化失败: {e}")
-            self.root.after(0, self.recognition_completed)
 
     def _create_vlm_provider(self) -> Optional[VLMProvider]:
         """根据配置创建 VLM Provider"""
@@ -759,6 +1022,14 @@ class MultiModalFoodApp:
             "fewshot_file_path": self.fewshot_file_path,
             "extra_labels": self.extra_labels,
             "extra_labels_file_path": self.extra_labels_file_path,
+            # 新增：开放式识别配置
+            "openset": {
+                "enabled": self.openset_enabled_var.get(),
+                "index_path": self.openset_index_path_var.get(),
+                "top_k": self.openset_top_k_var.get(),
+                "min_score": self.openset_min_score_var.get(),
+                "embedder_model": self.openset_embedder_model_var.get(),
+            },
         }
         
         try:
@@ -810,6 +1081,15 @@ class MultiModalFoodApp:
             if "extra_labels_file_path" in config:
                 self.extra_labels_file_path = config["extra_labels_file_path"]
             
+            # 新增：开放式识别配置
+            if "openset" in config:
+                openset_cfg = config["openset"]
+                self.openset_enabled_var.set(openset_cfg.get("enabled", False))
+                self.openset_index_path_var.set(openset_cfg.get("index_path", "exp_multimodal/dish_name_index"))
+                self.openset_top_k_var.set(openset_cfg.get("top_k", 3))
+                self.openset_min_score_var.set(openset_cfg.get("min_score", 0.5))
+                self.openset_embedder_model_var.set(openset_cfg.get("embedder_model", "quentinz/bge-large-zh-v1.5"))
+            
             self.refresh_extra_labels_view()
             self.refresh_fewshot_view()
             self.on_provider_change()
@@ -862,6 +1142,15 @@ class MultiModalFoodApp:
             if "extra_labels_file_path" in config:
                 self.extra_labels_file_path = config["extra_labels_file_path"]
             
+            # 新增：开放式识别配置
+            if "openset" in config:
+                openset_cfg = config["openset"]
+                self.openset_enabled_var.set(openset_cfg.get("enabled", False))
+                self.openset_index_path_var.set(openset_cfg.get("index_path", "exp_multimodal/dish_name_index"))
+                self.openset_top_k_var.set(openset_cfg.get("top_k", 3))
+                self.openset_min_score_var.set(openset_cfg.get("min_score", 0.5))
+                self.openset_embedder_model_var.set(openset_cfg.get("embedder_model", "quentinz/bge-large-zh-v1.5"))
+            
             self.refresh_extra_labels_view()
             self.refresh_fewshot_view()
             self.on_provider_change()

vlm_config.json

@@ -11023,5 +11023,12 @@
     "whole": [],
     "processed": []
   },
-  "extra_labels_file_path": "C:/Users/Huawei/Desktop/labels.json"
+  "extra_labels_file_path": "C:/Users/Huawei/Desktop/labels.json",
+  "openset": {
+    "enabled": true,
+    "index_path": "exp_multimodal/dish_name_index",
+    "top_k": 3,
+    "min_score": 0.5,
+    "embedder_model": "quentinz/bge-large-zh-v1.5"
+  }
 }