处理embedding的请求时长。
This commit is contained in:
@@ -6,6 +6,7 @@ import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from typing import List
|
||||
|
||||
import faiss
|
||||
@@ -38,18 +39,30 @@ def build_index(
|
||||
|
||||
print(f"[BuildIndex] Total dishes={len(dish_names)} batch_size={batch_size}")
|
||||
|
||||
# 批量编码
|
||||
# 批量编码(带重试机制)
|
||||
all_embeddings = []
|
||||
for i in range(0, len(dish_names), batch_size):
|
||||
batch = dish_names[i:i+batch_size]
|
||||
print(f"[BuildIndex] Encoding batch {i//batch_size + 1}/{(len(dish_names)-1)//batch_size + 1} (size={len(batch)})...")
|
||||
batch_num = i//batch_size + 1
|
||||
total_batches = (len(dish_names)-1)//batch_size + 1
|
||||
print(f"[BuildIndex] Encoding batch {batch_num}/{total_batches} (size={len(batch)})...")
|
||||
|
||||
try:
|
||||
batch_embs = embedder.encode(batch)
|
||||
all_embeddings.append(batch_embs)
|
||||
except Exception as e:
|
||||
print(f"[BuildIndex] Error encoding batch {i//batch_size + 1}: {e}")
|
||||
raise
|
||||
# 重试机制:最多3次,指数退避
|
||||
max_retries = 3
|
||||
for retry in range(max_retries):
|
||||
try:
|
||||
batch_embs = embedder.encode(batch)
|
||||
all_embeddings.append(batch_embs)
|
||||
break # 成功则跳出重试循环
|
||||
except Exception as e:
|
||||
if retry < max_retries - 1:
|
||||
wait_time = 5 * (retry + 1) # 5s, 10s, 15s
|
||||
print(f"[BuildIndex] Batch {batch_num} failed (attempt {retry+1}/{max_retries}): {e}")
|
||||
print(f"[BuildIndex] Retrying in {wait_time} seconds...")
|
||||
time.sleep(wait_time)
|
||||
else:
|
||||
print(f"[BuildIndex] Batch {batch_num} failed after {max_retries} attempts: {e}")
|
||||
raise
|
||||
|
||||
# 合并所有向量
|
||||
embeddings = np.vstack(all_embeddings)
|
||||
@@ -140,6 +153,7 @@ def main():
|
||||
embedder = OllamaEmbedder(
|
||||
base_url=args.embedder_url,
|
||||
model=args.embedder_model,
|
||||
timeout=180, # 3分钟超时
|
||||
)
|
||||
|
||||
# 构建索引
|
||||
|
||||
@@ -778,7 +778,7 @@ class MultiModalFoodApp:
|
||||
# 初始化 Embedder
|
||||
embedder_url = self.ollama_url_var.get().strip() or DEFAULT_OLLAMA_URL
|
||||
embedder_model = self.openset_embedder_model_var.get().strip()
|
||||
embedder = OllamaEmbedder(base_url=embedder_url, model=embedder_model)
|
||||
embedder = OllamaEmbedder(base_url=embedder_url, model=embedder_model, timeout=180)
|
||||
|
||||
# 构建索引(使用合并后的菜品列表)
|
||||
output_dir = self.openset_index_path_var.get()
|
||||
|
||||
@@ -497,16 +497,17 @@ def main():
|
||||
# 配置参数
|
||||
# MODEL_PATH = "../model/embedding_20251011_133653/best_embedding_model.pth"
|
||||
# MODEL_PATH = "../model/ProcessedIngredientRecognition/embedding_20251103_172012/best_embedding_model.pth"
|
||||
# MODEL_PATH = "../model/WholeIngredientRecognition/cosface_20251106_134718/best_cosface_model.pth"
|
||||
MODEL_PATH = "../model/WholeIngredientRecognition/cosface_20251113_160103/best_cosface_model.pth"
|
||||
# MODEL_PATH = "../model/WholeIngredientRecognition/grid_search_20251113_095659/model_s68.0_m0.4.pth"
|
||||
# MODEL_PATH = "../model/DishClassification/cosface_20251105_200551/best_embedding_model.pth"
|
||||
MODEL_PATH = "../model/DishClassification/cosface_20251111_153649/best_cosface_model.pth"
|
||||
# MODEL_PATH = "../model/DishClassification/cosface_20251111_153649/best_cosface_model.pth"
|
||||
# TRAIN_DIR = "../dataset/ProcessedIngredientRecognition/train"
|
||||
# TRAIN_DIR = "../dataset/WholeIngredientRecognition/train"
|
||||
TRAIN_DIR = "../dataset/DishClassification/train"
|
||||
TRAIN_DIR = "../dataset/WholeIngredientRecognition/train"
|
||||
# TRAIN_DIR = "../dataset/DishClassification/train"
|
||||
|
||||
# OUTPUT_DIR = "ProcessedIngredientRecognition/faiss_index"
|
||||
# OUTPUT_DIR = "WholeIngredientRecognition/faiss_index"
|
||||
OUTPUT_DIR = "DishClassification/faiss_index"
|
||||
OUTPUT_DIR = "WholeIngredientRecognition/faiss_index"
|
||||
# OUTPUT_DIR = "DishClassification/faiss_index"
|
||||
BATCH_SIZE = 16
|
||||
INDEX_TYPE = 'flat' # 'flat', 'ivf', 'hnsw'
|
||||
EMBEDDING_DIM = 512
|
||||
|
||||
@@ -68,8 +68,8 @@ TASKS = {
|
||||
batch_size=64,
|
||||
lr=8e-4,
|
||||
aug_strength='medium',
|
||||
cosface_s=64.0, # 完整食材:类间区分度高,使用标准scale
|
||||
cosface_m=0.38, # 较大margin,强化类间分离(番茄vs土豆差异明显)
|
||||
cosface_s=68.0, # 完整食材:类间区分度高,使用标准scale
|
||||
cosface_m=0.35, # 较大margin,强化类间分离(番茄vs土豆差异明显)
|
||||
),
|
||||
'processed_ingredient': TaskConfig(
|
||||
name='ProcessedIngredientRecognition',
|
||||
@@ -422,8 +422,8 @@ def main(task_key: str = 'dish', s: Optional[float] = None, m: Optional[float] =
|
||||
if __name__ == '__main__':
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('task', choices=list(TASKS.keys()), nargs='?', default='dish')
|
||||
# parser.add_argument('task', choices=list(TASKS.keys()), nargs='?', default='whole_ingredient')
|
||||
# parser.add_argument('task', choices=list(TASKS.keys()), nargs='?', default='dish')
|
||||
parser.add_argument('task', choices=list(TASKS.keys()), nargs='?', default='whole_ingredient')
|
||||
parser.add_argument('--s', type=float, default=None, help='CosFace scale factor (默认使用任务配置值)')
|
||||
parser.add_argument('--m', type=float, default=None, help='CosFace margin (默认使用任务配置值)')
|
||||
# parser.add_argument('--epochs', type=int, default=60)
|
||||
|
||||
Reference in New Issue
Block a user