增加多模态模型识别功能。
This commit is contained in:
@@ -0,0 +1,3 @@
|
||||
# Marks this directory as a Python package for relative imports
|
||||
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"番茄": "西红柿",
|
||||
"西红柿": "西红柿",
|
||||
"胡瓜": "黄瓜",
|
||||
"马铃薯": "土豆",
|
||||
"洋芋": "土豆",
|
||||
"芹菜(茎)": "芹菜",
|
||||
"蒜薹": "蒜薹",
|
||||
"蒜苔": "蒜薹",
|
||||
"辣椒(青,尖)": "辣椒",
|
||||
"辣椒(青，尖)": "辣椒"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from typing import Dict
|
||||
|
||||
# 兼容：支持直接运行脚本或用 -m 模块方式运行
|
||||
if __name__ == "__main__" and __package__ is None:
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
# 统一使用绝对导入，避免相对导入在脚本直跑时失败
|
||||
from exp_multimodal.labels import build_labels
|
||||
from exp_multimodal.vlm_classifier import classify_image
|
||||
from exp_multimodal.ollama_client import OLLAMA_URL, DEFAULT_MODEL
|
||||
|
||||
|
||||
FEWSHOT_HINTS: Dict[str, str] = {
|
||||
# 可选：仅对菜品模式提供少量文字提示，帮助区分相似菜
|
||||
# "西红柿鸡蛋": "红黄对比明显，蛋块与番茄块同炒，汤汁偏红",
|
||||
# "麻婆豆腐": "豆腐块+红油辣椒+花椒，肉末点缀",
|
||||
}
|
||||
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--mode", choices=["dish", "whole", "processed"], default="dish")
|
||||
ap.add_argument("--image", default=r"D:\MyProjects\PythonProjects\FoodClassifier\dataset\DishClassification\test\回锅肉\img01.png")
|
||||
ap.add_argument("--alias_map", default=None)
|
||||
|
||||
args = ap.parse_args()
|
||||
|
||||
print(
|
||||
f"[Main] mode={args.mode} image={args.image} alias_map={args.alias_map} "
|
||||
f"env_OLLAMA_URL={os.getenv('OLLAMA_URL')} env_VLM_MODEL={os.getenv('VLM_MODEL')} "
|
||||
f"defaults url={OLLAMA_URL} model={DEFAULT_MODEL}"
|
||||
)
|
||||
print(f"[Main] Image exists={os.path.exists(args.image)} size={os.path.getsize(args.image) if os.path.exists(args.image) else 'N/A'}")
|
||||
|
||||
labels = build_labels(args.mode, args.alias_map)
|
||||
ingredient_only = args.mode in {"whole", "processed"}
|
||||
fewshot = FEWSHOT_HINTS if args.mode == "dish" else None
|
||||
print(f"[Main] Built labels count={len(labels)} ingredient_only={ingredient_only} fewshot={bool(fewshot)}")
|
||||
|
||||
res = classify_image(
|
||||
args.image,
|
||||
labels,
|
||||
fewshot_hints=fewshot,
|
||||
ingredient_only=ingredient_only,
|
||||
)
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
{"mode": args.mode, "result": res, "num_labels": len(labels)},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from typing import Dict, List
|
||||
|
||||
|
||||
def _list_dirs(path: str) -> List[str]:
|
||||
if not os.path.isdir(path):
|
||||
return []
|
||||
return [d for d in os.listdir(path) if os.path.isdir(os.path.join(path, d))]
|
||||
|
||||
|
||||
def _normalize(name: str) -> str:
|
||||
# 去括号/方括号内注释，统一去空格
|
||||
name = re.sub(r"[（(].*?[)）]", "", name)
|
||||
name = re.sub(r"\[.*?\]", "", name)
|
||||
return name.replace(" ", "").strip()
|
||||
|
||||
|
||||
_SHAPE_COOKING_TERMS = [
|
||||
"切丝",
|
||||
"切片",
|
||||
"切丁",
|
||||
"切段",
|
||||
"切块",
|
||||
"切条",
|
||||
"切末",
|
||||
"切花",
|
||||
"丝",
|
||||
"片",
|
||||
"丁",
|
||||
"段",
|
||||
"块",
|
||||
"条",
|
||||
"末",
|
||||
"清炒",
|
||||
"红烧",
|
||||
"爆炒",
|
||||
"炒",
|
||||
"煮",
|
||||
"炖",
|
||||
"蒸",
|
||||
"炸",
|
||||
"拌",
|
||||
"焯",
|
||||
"烤",
|
||||
"煎",
|
||||
"烩",
|
||||
"烫",
|
||||
]
|
||||
|
||||
|
||||
def _base_ingredient(name: str) -> str:
|
||||
n = _normalize(name)
|
||||
for kw in _SHAPE_COOKING_TERMS:
|
||||
n = n.replace(kw, "")
|
||||
n = re.sub(r"[、，,·\-/]", "", n)
|
||||
return n.strip()
|
||||
|
||||
|
||||
def load_alias_map(fp: str | None = None) -> Dict[str, str]:
|
||||
if fp and os.path.isfile(fp):
|
||||
with open(fp, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
return {}
|
||||
|
||||
|
||||
def build_labels(mode: str, alias_map_path: str | None = None) -> List[str]:
|
||||
# 使 dataset 路径与运行目录无关：基于项目根目录（exp_multimodal 的上一级）
|
||||
base = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "dataset")
|
||||
if mode == "dish":
|
||||
root = os.path.join(base, "DishClassification", "train")
|
||||
elif mode == "whole":
|
||||
root = os.path.join(base, "WholeIngredientRecognition", "train")
|
||||
elif mode == "processed":
|
||||
root = os.path.join(base, "ProcessedIngredientRecognition", "train")
|
||||
else:
|
||||
return []
|
||||
|
||||
raw = _list_dirs(root)
|
||||
alias = load_alias_map(alias_map_path)
|
||||
std: List[str] = []
|
||||
for r in raw:
|
||||
n = _normalize(r)
|
||||
n = alias.get(n, n)
|
||||
if mode in ("whole", "processed"):
|
||||
n = _base_ingredient(n)
|
||||
if n:
|
||||
std.append(n)
|
||||
|
||||
# 去重并排序
|
||||
return sorted(list(dict.fromkeys(std)))
|
||||
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
import base64
|
||||
import os
|
||||
import time
|
||||
from typing import Any, Dict, List
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
OLLAMA_URL = os.getenv("OLLAMA_URL", "http://192.168.1.250:11434")
|
||||
DEFAULT_MODEL = os.getenv("VLM_MODEL", "qwen2.5vl:32b")
|
||||
|
||||
|
||||
def _img_to_data_uri(path: str) -> str:
|
||||
with open(path, "rb") as f:
|
||||
b64 = base64.b64encode(f.read()).decode("utf-8")
|
||||
return f"data:image/jpeg;base64,{b64}"
|
||||
|
||||
|
||||
def chat_vision(
|
||||
prompt: str,
|
||||
image_paths: List[str],
|
||||
*,
|
||||
temperature: float = 0.1,
|
||||
options: Dict[str, Any] | None = None,
|
||||
model: str = DEFAULT_MODEL,
|
||||
timeout_sec: int = 120,
|
||||
) -> str:
|
||||
# Logging: request summary before heavy work
|
||||
try:
|
||||
img_sizes = []
|
||||
for p in image_paths:
|
||||
try:
|
||||
img_sizes.append(os.path.getsize(p))
|
||||
except Exception:
|
||||
img_sizes.append(-1)
|
||||
print(
|
||||
f"[Ollama] Prepare chat url={OLLAMA_URL}/api/chat model={model} temp={temperature} "
|
||||
f"images={len(image_paths)} sizes={img_sizes} timeout={timeout_sec}s"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
images = [_img_to_data_uri(p) for p in image_paths]
|
||||
payload: Dict[str, Any] = {
|
||||
"model": model,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": prompt,
|
||||
"images": images,
|
||||
}
|
||||
],
|
||||
"options": {"temperature": temperature, **(options or {})},
|
||||
"stream": False,
|
||||
}
|
||||
|
||||
t0 = time.time()
|
||||
try:
|
||||
resp = requests.post(f"{OLLAMA_URL}/api/chat", json=payload, timeout=timeout_sec)
|
||||
dt = time.time() - t0
|
||||
print(f"[Ollama] Response status={resp.status_code} elapsed={dt:.2f}s")
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
text = (data.get("message") or {}).get("content", "")
|
||||
_preview = (text[:120]).replace("\n", " ")
|
||||
print(f"[Ollama] Text length={len(text)} preview={_preview}...")
|
||||
return text
|
||||
except Exception as e:
|
||||
dt = time.time() - t0
|
||||
print(f"[Ollama][Error] during request elapsed={dt:.2f}s: {e}")
|
||||
raise
|
||||
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
|
||||
def build_closedset_prompt(
|
||||
labels: List[str],
|
||||
fewshot_hints: Optional[Dict[str, str]] = None,
|
||||
ingredient_only: bool = False,
|
||||
) -> str:
|
||||
label_str = "，".join(labels)
|
||||
|
||||
hint_lines: List[str] = []
|
||||
if fewshot_hints:
|
||||
hint_lines.append("以下是少量文字示例帮助你区分：")
|
||||
for k, v in list(fewshot_hints.items())[:20]:
|
||||
hint_lines.append(f"- {k}：{v}")
|
||||
hints = "\n".join(hint_lines)
|
||||
|
||||
constraint = (
|
||||
"你只需输出食材名称，不要包含任何烹饪方式或形态描述（如切丝/切片/炒/煮/红烧/清炒等）。\n"
|
||||
if ingredient_only
|
||||
else ""
|
||||
)
|
||||
|
||||
return (
|
||||
"你是食品图像分类专家。请严格在给定的标签集合中选择最符合图片内容的一个标签。\n"
|
||||
"只允许返回一个标签，禁止输出集合外的词。请同时给出0-1置信度。\n"
|
||||
f"{constraint}"
|
||||
f"标签集合（封闭集，中文）：{label_str}\n"
|
||||
f"{hints}\n"
|
||||
'请仅输出合法 JSON：{"label":"<标签>","confidence":0-1}\n'
|
||||
"不要输出任何额外解释或文本。"
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import json
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from .ollama_client import chat_vision
|
||||
from .prompts import build_closedset_prompt
|
||||
|
||||
|
||||
def classify_image(
|
||||
image_path: str,
|
||||
labels: List[str],
|
||||
fewshot_hints: Optional[Dict[str, str]] = None,
|
||||
ingredient_only: bool = False,
|
||||
) -> Dict:
|
||||
if not labels:
|
||||
print("[VLM] Empty labels provided, return Unknown")
|
||||
return {"label": "Unknown", "confidence": 0.0}
|
||||
|
||||
print(
|
||||
f"[VLM] Start classify image={image_path} labels={len(labels)} "
|
||||
f"ingredient_only={ingredient_only} fewshot={bool(fewshot_hints)}"
|
||||
)
|
||||
prompt = build_closedset_prompt(labels, fewshot_hints, ingredient_only)
|
||||
_preview_prompt = (prompt[:120]).replace("\n", " ")
|
||||
print(f"[VLM] Prompt length={len(prompt)} preview={_preview_prompt}...")
|
||||
|
||||
print("[VLM] Calling chat_vision...")
|
||||
text = chat_vision(prompt, [image_path], temperature=0.1)
|
||||
_preview_text = (text[:120]).replace("\n", " ")
|
||||
print(f"[VLM] Received text length={len(text)} preview={_preview_text}...")
|
||||
|
||||
# 主路径：严格 JSON 解析
|
||||
try:
|
||||
obj = json.loads(text)
|
||||
if isinstance(obj, dict) and "label" in obj:
|
||||
label = obj.get("label", "")
|
||||
# 纠偏：若输出不在封闭集，尝试包含匹配；否则 Unknown
|
||||
if label not in labels:
|
||||
for lb in labels:
|
||||
if lb in str(label):
|
||||
label = lb
|
||||
break
|
||||
else:
|
||||
label = "Unknown"
|
||||
conf = obj.get("confidence", 0.0)
|
||||
try:
|
||||
conf = float(conf)
|
||||
except Exception:
|
||||
conf = 0.0
|
||||
result = {"label": label, "confidence": conf}
|
||||
print(f"[VLM] Parsed JSON result={result}")
|
||||
return result
|
||||
except Exception as e:
|
||||
print(f"[VLM] JSON parse failed: {e}")
|
||||
|
||||
# 回退：字符串包含匹配
|
||||
for lb in labels:
|
||||
if lb in text:
|
||||
print(f"[VLM] Fallback match label={lb}")
|
||||
return {"label": lb, "confidence": 0.5}
|
||||
|
||||
print("[VLM] No match, return Unknown")
|
||||
return {"label": "Unknown", "confidence": 0.0}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user