diff --git a/classifier/embedding_food_classifier_app.py b/classifier/embedding_food_classifier_app.py index 4b9a460..b20ea27 100644 --- a/classifier/embedding_food_classifier_app.py +++ b/classifier/embedding_food_classifier_app.py @@ -65,13 +65,13 @@ class EmbeddingFoodClassifierApp: BASE_DIR = os.path.dirname(os.path.abspath(__file__)) # model_path = "../model/embedding_20251011_133653/best_embedding_model.pth" # model_path = os.path.join(BASE_DIR, "../model/embedding_20251011_133653/best_embedding_model.pth") - model_path = os.path.join(BASE_DIR, "../model/WholeIngredientRecognition/embedding_20251021_085915/best_embedding_model.pth") + model_path = os.path.join(BASE_DIR, "../model/WholeIngredientRecognition/embedding_20251023_120531/best_embedding_model.pth") # model_path = os.path.join(BASE_DIR, "../model/DishClassification/embedding_20251022_093635/best_embedding_model.pth") # FAISS索引目录 # index_dir = "../faiss_vector_db/faiss_index" # index_dir = os.path.join(BASE_DIR, "../faiss_vector_db/faiss_index") - index_dir = os.path.join(BASE_DIR, "../faiss_vector_db/WholeIngredientRecognition/faiss_index") + index_dir = os.path.join(BASE_DIR, "../faiss_vector_db/WholeIngredientRecognition/faiss_index102402") # index_dir = os.path.join(BASE_DIR, "../faiss_vector_db/DishClassification/faiss_index") if os.path.exists(model_path) and os.path.exists(index_dir): @@ -1087,12 +1087,13 @@ class EmbeddingFoodClassifierApp: """识别所有图片""" try: self.current_results.clear() - + score_list = [] for i, img_info in enumerate(self.uploaded_images): # 使用embedding相似度识别 if self.model is not None and self.faiss_index is not None: # 使用真实的embedding模型和FAISS索引 - predicted_class, confidence, similar_images = self.predict_with_embedding(img_info['image']) + predicted_class, confidence, similar_images,score = self.predict_with_embedding(img_info['image']) + score_list.append(score) else: # 模拟预测结果 predicted_class = np.random.choice(self.class_names) @@ -1122,7 +1123,7 @@ class EmbeddingFoodClassifierApp: # 更新UI（在主线程中） self.root.after(0, self.update_progress, i + 1, len(self.uploaded_images)) - + print('socre_list',score_list) # 识别完成，更新UI self.root.after(0, self.recognition_completed) @@ -1144,9 +1145,11 @@ class EmbeddingFoodClassifierApp: # 在FAISS索引中搜索最相似的k张图片 scores, indices = self.faiss_index.search(query_embedding, k) - print("最相似的图片索引:", indices) - print("最相似的图片分数:", scores) - + # print("最相似的图片索引:", indices) + # print("最相似的图片分数:", type(scores[0])) + # print("最相似的图片分数:", scores[0,0]) + # print("最相似的图片分数:", type(scores[0,0])) + # 收集相似图片的类别 similar_classes = [] similar_images = [] @@ -1179,12 +1182,12 @@ class EmbeddingFoodClassifierApp: vote_ratio = class_counts[predicted_class] / len(similar_classes) confidence = max_score * vote_ratio - return predicted_class, confidence, similar_images + return predicted_class, confidence, similar_images,round(scores[0,0],3) else: # 如果没有找到相似图片，随机选择一个类别 predicted_class = np.random.choice(self.class_names) confidence = 0.1 - return predicted_class, confidence, [] + return predicted_class, confidence, [],scores[0,0] except Exception as e: print(f"Embedding预测出错: {e}") diff --git a/faiss_vector_db/build_faiss_index.py b/faiss_vector_db/build_faiss_index.py index 7f01be5..0ad608f 100644 --- a/faiss_vector_db/build_faiss_index.py +++ b/faiss_vector_db/build_faiss_index.py @@ -480,12 +480,12 @@ def main(): """主函数""" # 配置参数 # MODEL_PATH = "../model/embedding_20251011_133653/best_embedding_model.pth" - # MODEL_PATH = "../model/WholeIngredientRecognition/embedding_20251021_085915/best_embedding_model.pth" - MODEL_PATH = "../model/DishClassification/embedding_20251022_093635/best_embedding_model.pth" - # TRAIN_DIR = "../dataset/WholeIngredientRecognition/train" - TRAIN_DIR = "../dataset/DishClassification/train" - # OUTPUT_DIR = "WholeIngredientRecognition/faiss_index" - OUTPUT_DIR = "DishClassification/faiss_index" + MODEL_PATH = "../model/WholeIngredientRecognition/embedding_20251021_085915/best_embedding_model.pth" + # MODEL_PATH = "../model/DishClassification/embedding_20251022_093635/best_embedding_model.pth" + TRAIN_DIR = "../dataset/WholeIngredientRecognition/train" + # TRAIN_DIR = "../dataset/DishClassification/train" + OUTPUT_DIR = "WholeIngredientRecognition/faiss_index" + # OUTPUT_DIR = "DishClassification/faiss_index" BATCH_SIZE = 16 INDEX_TYPE = 'flat' # 'flat', 'ivf', 'hnsw' EMBEDDING_DIM = 512 diff --git a/faiss_vector_db/visualize_embeddings.py b/faiss_vector_db/visualize_embeddings.py index 4f0f60d..1e67dc3 100644 --- a/faiss_vector_db/visualize_embeddings.py +++ b/faiss_vector_db/visualize_embeddings.py @@ -162,10 +162,10 @@ def plot_2d(Z: np.ndarray, y: np.ndarray, title: str, out_path: Optional[str] = def main(): parser = argparse.ArgumentParser(description="可视化高维 embedding 并进行坍塌诊断") - parser.add_argument("--embeddings", type=str, default=os.path.join("DishClassification/faiss_index", "embeddings.json"), help="embeddings.json 路径") - # parser.add_argument("--embeddings", type=str, default=os.path.join("WholeIngredientRecognition/faiss_index", "embeddings.json"), help="embeddings.json 路径") - parser.add_argument("--labels", type=str, default=os.path.join("DishClassification/faiss_index", "labels.json"), help="labels.json 路径") - # parser.add_argument("--labels", type=str, default=os.path.join("WholeIngredientRecognition/faiss_index", "labels.json"), help="labels.json 路径") + # parser.add_argument("--embeddings", type=str, default=os.path.join("DishClassification/faiss_index", "embeddings.json"), help="embeddings.json 路径") + parser.add_argument("--embeddings", type=str, default=os.path.join("WholeIngredientRecognition/faiss_index", "embeddings.json"), help="embeddings.json 路径") + # parser.add_argument("--labels", type=str, default=os.path.join("DishClassification/faiss_index", "labels.json"), help="labels.json 路径") + parser.add_argument("--labels", type=str, default=os.path.join("WholeIngredientRecognition/faiss_index", "labels.json"), help="labels.json 路径") parser.add_argument("--method", type=str, default="pca", choices=["pca", "tsne", "umap"], help="降维方法") parser.add_argument("--seed", type=int, default=42) parser.add_argument("--max_points", type=int, default=None, help="抽样上限，避免t-SNE/UMAP过慢；None为全量") diff --git a/train/train_embedding.py b/train/train_embedding.py index facae97..e77d97a 100644 --- a/train/train_embedding.py +++ b/train/train_embedding.py @@ -32,6 +32,7 @@ class TaskConfig: name: str train_dir: str val_dir: str + unknown_dir: str # 未知类别数据目录 embedding_dim: int batch_size: int lr: float @@ -45,6 +46,7 @@ TASKS = { name="DishClassification", train_dir=os.path.join(settings.BASE_DIR, "dataset", "DishClassification", "train"), val_dir=os.path.join(settings.BASE_DIR, "dataset", "DishClassification", "val"), + unknown_dir=os.path.join(settings.BASE_DIR, "dataset", "Unknown", "DishClassification","train"), # 未知类别食物目录 embedding_dim=512, batch_size=16, lr=1e-3, @@ -58,6 +60,7 @@ TASKS = { name="WholeIngredientRecognition", train_dir=os.path.join(settings.BASE_DIR, "dataset", "WholeIngredientRecognition", "train"), val_dir=os.path.join(settings.BASE_DIR, "dataset", "WholeIngredientRecognition", "val"), + unknown_dir=os.path.join(settings.BASE_DIR, "dataset", "Unknown","WholeIngredientRecognition", "train"), # 未知类别食物目录 embedding_dim=512, batch_size=32, lr=8e-4, @@ -70,6 +73,7 @@ TASKS = { name="ProcessedIngredientRecognition", train_dir=os.path.join(settings.BASE_DIR, "dataset", "ProcessedIngredientRecognition", "train"), val_dir=os.path.join(settings.BASE_DIR, "dataset", "ProcessedIngredientRecognition", "val"), + unknown_dir=os.path.join(settings.BASE_DIR, "dataset", "Unknown", "ProcessedIngredientRecognition","train"), # 未知类别食物目录 embedding_dim=512, batch_size=16, lr=1e-3, @@ -142,27 +146,33 @@ class TripletDataset(Dataset): 三元组数据集，用于三元组损失训练 每个样本包含：锚点(anchor)、正样本(positive)、负样本(negative) """ - - def __init__(self, dataset_path: str, transform=None, samples_per_class: int = 100): + + def __init__(self, dataset_path: str, transform=None, samples_per_class: int = 100, unknown_dir: str = None, unknown_prob: float = 0.7): """ 初始化三元组数据集 - + Args: dataset_path: 数据集路径 transform: 数据变换 samples_per_class: 每个类别最多使用的样本数 + unknown_dir: 未知类别数据目录 + unknown_prob: 使用未知类别作为负样本的概率 """ self.dataset_path = dataset_path self.transform = transform self.samples_per_class = samples_per_class - + self.unknown_dir = unknown_dir + self.unknown_prob = unknown_prob + # 加载数据集 self.class_to_idx = {} self.idx_to_class = {} self.samples_by_class = defaultdict(list) self.all_samples = [] - + self.unknown_samples = [] # 未知类别样本列表 + self._load_dataset() + self._load_unknown_samples() def _load_dataset(self): """加载数据集并按类别组织""" @@ -191,24 +201,39 @@ class TripletDataset(Dataset): logger.info(f"加载数据集完成:") for class_name, class_idx in self.class_to_idx.items(): logger.info(f" {class_name}: {len(self.samples_by_class[class_idx])} 张图片") - + + def _load_unknown_samples(self): + """加载未知类别样本""" + if not self.unknown_dir or not os.path.exists(self.unknown_dir): + logger.info("未指定未知类别目录或目录不存在，跳过未知样本加载") + return + + # 遍历未知类别目录下的所有图片 + for root, dirs, files in os.walk(self.unknown_dir): + for file in files: + if file.lower().endswith(('.png', '.jpg', '.jpeg')): + img_path = os.path.join(root, file) + self.unknown_samples.append(img_path) + + logger.info(f"加载未知类别样本完成: {len(self.unknown_samples)} 张图片") + def __len__(self): return len(self.all_samples) - + def __getitem__(self, idx): """ 获取三元组样本 - + Returns: tuple: (anchor, positive, negative, anchor_label) """ # 获取锚点样本 anchor_path, anchor_label = self.all_samples[idx] anchor_img = self._load_image(anchor_path) - + # 获取正样本（同类别的不同图片） # 需要从自己所在类别中先把自己给排除掉 - positive_candidates = [path for path in self.samples_by_class[anchor_label] + positive_candidates = [path for path in self.samples_by_class[anchor_label] if path != anchor_path] if positive_candidates: # 从这里可以看出来是随机选的 @@ -216,17 +241,42 @@ class TripletDataset(Dataset): else: positive_path = anchor_path # 如果只有一张图片，使用自己作为正样本 positive_img = self._load_image(positive_path) - - # 获取负样本（不同类别的图片） + + # 获取负样本（使用策略选择未知类别或已知类别） + negative_img = self._get_negative_sample() + + return anchor_img, positive_img, negative_img, anchor_label + + def _get_negative_sample(self): + """ + 获取负样本，支持未知类别和已知类别的混合策略 + + Returns: + torch.Tensor: 负样本图片 + """ + # 如果有未知类别样本且概率满足条件，优先选择未知类别 + if self.unknown_samples and random.random() < self.unknown_prob: + # 选择未知类别作为负样本 + negative_path = random.choice(self.unknown_samples) + return self._load_image(negative_path) + + # 否则选择已知类别作为负样本（原来的逻辑） + # 为了安全起见，选择一个随机锚点 + anchor_idx = random.randint(0, len(self.all_samples) - 1) + anchor_path, anchor_label = self.all_samples[anchor_idx] + # 获得其它类别 negative_classes = [cls for cls in self.samples_by_class.keys() if cls != anchor_label] - # 随便选一个类别 - negative_class = random.choice(negative_classes) - # 随便选一个路径 - negative_path = random.choice(self.samples_by_class[negative_class]) - negative_img = self._load_image(negative_path) - - return anchor_img, positive_img, negative_img, anchor_label + if negative_classes: + # 随便选一个类别 + negative_class = random.choice(negative_classes) + # 随便选一个路径 + negative_path = random.choice(self.samples_by_class[negative_class]) + else: + # 如果没有其他类别，使用自己作为负样本（这种情况应该很少见） + negative_path = anchor_path + + return self._load_image(negative_path) def _load_image(self, image_path: str): """加载并预处理图片""" @@ -644,13 +694,17 @@ def main(task_key: str = "dish"): train_dataset = TripletDataset( dataset_path=cfg.train_dir, transform=transform_train, - samples_per_class=550 + samples_per_class=550, + unknown_dir=cfg.unknown_dir, + unknown_prob=0.7 # 70%概率使用未知类别作为负样本 ) - + val_dataset = TripletDataset( dataset_path=cfg.val_dir, transform=transform_val, - samples_per_class=50 + samples_per_class=50, + unknown_dir=cfg.unknown_dir, + unknown_prob=0.7 # 70%概率使用未知类别作为负样本 ) # 创建数据加载器 @@ -818,5 +872,5 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("task", choices=list(TASKS.keys()), nargs="?", default="dish") args = parser.parse_args() - # main("whole_ingredient") - main("dish") \ No newline at end of file + main("whole_ingredient") + # main("dish") \ No newline at end of file