处理embedding的请求时长。

This commit is contained in:
2025-11-13 16:23:02 +08:00
parent d24b40c9ec
commit 3ae8c677ff
4 changed files with 34 additions and 19 deletions
+22 -8
View File
@@ -6,6 +6,7 @@ import argparse
import json
import os
import sys
import time
from typing import List
import faiss
@@ -38,18 +39,30 @@ def build_index(
print(f"[BuildIndex] Total dishes={len(dish_names)} batch_size={batch_size}")
# 批量编码
# 批量编码(带重试机制)
all_embeddings = []
for i in range(0, len(dish_names), batch_size):
batch = dish_names[i:i+batch_size]
print(f"[BuildIndex] Encoding batch {i//batch_size + 1}/{(len(dish_names)-1)//batch_size + 1} (size={len(batch)})...")
batch_num = i//batch_size + 1
total_batches = (len(dish_names)-1)//batch_size + 1
print(f"[BuildIndex] Encoding batch {batch_num}/{total_batches} (size={len(batch)})...")
try:
batch_embs = embedder.encode(batch)
all_embeddings.append(batch_embs)
except Exception as e:
print(f"[BuildIndex] Error encoding batch {i//batch_size + 1}: {e}")
raise
# 重试机制:最多3次,指数退避
max_retries = 3
for retry in range(max_retries):
try:
batch_embs = embedder.encode(batch)
all_embeddings.append(batch_embs)
break # 成功则跳出重试循环
except Exception as e:
if retry < max_retries - 1:
wait_time = 5 * (retry + 1) # 5s, 10s, 15s
print(f"[BuildIndex] Batch {batch_num} failed (attempt {retry+1}/{max_retries}): {e}")
print(f"[BuildIndex] Retrying in {wait_time} seconds...")
time.sleep(wait_time)
else:
print(f"[BuildIndex] Batch {batch_num} failed after {max_retries} attempts: {e}")
raise
# 合并所有向量
embeddings = np.vstack(all_embeddings)
@@ -140,6 +153,7 @@ def main():
embedder = OllamaEmbedder(
base_url=args.embedder_url,
model=args.embedder_model,
timeout=180, # 3分钟超时
)
# 构建索引
+1 -1
View File
@@ -778,7 +778,7 @@ class MultiModalFoodApp:
# 初始化 Embedder
embedder_url = self.ollama_url_var.get().strip() or DEFAULT_OLLAMA_URL
embedder_model = self.openset_embedder_model_var.get().strip()
embedder = OllamaEmbedder(base_url=embedder_url, model=embedder_model)
embedder = OllamaEmbedder(base_url=embedder_url, model=embedder_model, timeout=180)
# 构建索引(使用合并后的菜品列表)
output_dir = self.openset_index_path_var.get()
+7 -6
View File
@@ -497,16 +497,17 @@ def main():
# 配置参数
# MODEL_PATH = "../model/embedding_20251011_133653/best_embedding_model.pth"
# MODEL_PATH = "../model/ProcessedIngredientRecognition/embedding_20251103_172012/best_embedding_model.pth"
# MODEL_PATH = "../model/WholeIngredientRecognition/cosface_20251106_134718/best_cosface_model.pth"
MODEL_PATH = "../model/WholeIngredientRecognition/cosface_20251113_160103/best_cosface_model.pth"
# MODEL_PATH = "../model/WholeIngredientRecognition/grid_search_20251113_095659/model_s68.0_m0.4.pth"
# MODEL_PATH = "../model/DishClassification/cosface_20251105_200551/best_embedding_model.pth"
MODEL_PATH = "../model/DishClassification/cosface_20251111_153649/best_cosface_model.pth"
# MODEL_PATH = "../model/DishClassification/cosface_20251111_153649/best_cosface_model.pth"
# TRAIN_DIR = "../dataset/ProcessedIngredientRecognition/train"
# TRAIN_DIR = "../dataset/WholeIngredientRecognition/train"
TRAIN_DIR = "../dataset/DishClassification/train"
TRAIN_DIR = "../dataset/WholeIngredientRecognition/train"
# TRAIN_DIR = "../dataset/DishClassification/train"
# OUTPUT_DIR = "ProcessedIngredientRecognition/faiss_index"
# OUTPUT_DIR = "WholeIngredientRecognition/faiss_index"
OUTPUT_DIR = "DishClassification/faiss_index"
OUTPUT_DIR = "WholeIngredientRecognition/faiss_index"
# OUTPUT_DIR = "DishClassification/faiss_index"
BATCH_SIZE = 16
INDEX_TYPE = 'flat' # 'flat', 'ivf', 'hnsw'
EMBEDDING_DIM = 512
+4 -4
View File
@@ -68,8 +68,8 @@ TASKS = {
batch_size=64,
lr=8e-4,
aug_strength='medium',
cosface_s=64.0, # 完整食材:类间区分度高,使用标准scale
cosface_m=0.38, # 较大margin,强化类间分离(番茄vs土豆差异明显)
cosface_s=68.0, # 完整食材:类间区分度高,使用标准scale
cosface_m=0.35, # 较大margin,强化类间分离(番茄vs土豆差异明显)
),
'processed_ingredient': TaskConfig(
name='ProcessedIngredientRecognition',
@@ -422,8 +422,8 @@ def main(task_key: str = 'dish', s: Optional[float] = None, m: Optional[float] =
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('task', choices=list(TASKS.keys()), nargs='?', default='dish')
# parser.add_argument('task', choices=list(TASKS.keys()), nargs='?', default='whole_ingredient')
# parser.add_argument('task', choices=list(TASKS.keys()), nargs='?', default='dish')
parser.add_argument('task', choices=list(TASKS.keys()), nargs='?', default='whole_ingredient')
parser.add_argument('--s', type=float, default=None, help='CosFace scale factor (默认使用任务配置值)')
parser.add_argument('--m', type=float, default=None, help='CosFace margin (默认使用任务配置值)')
# parser.add_argument('--epochs', type=int, default=60)