From 9e12a5fbc3e362ec22fa24ae193565ce9651df00 Mon Sep 17 00:00:00 2001 From: zhangpu <1250681871@qq.com> Date: Mon, 3 Nov 2025 09:29:57 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=A4=9A=E6=A8=A1=E6=80=81?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E8=AF=86=E5=88=AB=E5=8A=9F=E8=83=BD=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- exp_multimodal/__init__.py | 3 + exp_multimodal/alias_map.example.json | 13 ++++ exp_multimodal/exp_multimodal2.py | 63 ++++++++++++++++++ exp_multimodal/labels.py | 94 +++++++++++++++++++++++++++ exp_multimodal/ollama_client.py | 73 +++++++++++++++++++++ exp_multimodal/prompts.py | 34 ++++++++++ exp_multimodal/vlm_classifier.py | 64 ++++++++++++++++++ 7 files changed, 344 insertions(+) create mode 100644 exp_multimodal/__init__.py create mode 100644 exp_multimodal/alias_map.example.json create mode 100644 exp_multimodal/exp_multimodal2.py create mode 100644 exp_multimodal/labels.py create mode 100644 exp_multimodal/ollama_client.py create mode 100644 exp_multimodal/prompts.py create mode 100644 exp_multimodal/vlm_classifier.py diff --git a/exp_multimodal/__init__.py b/exp_multimodal/__init__.py new file mode 100644 index 0000000..694be0a --- /dev/null +++ b/exp_multimodal/__init__.py @@ -0,0 +1,3 @@ +# Marks this directory as a Python package for relative imports + + diff --git a/exp_multimodal/alias_map.example.json b/exp_multimodal/alias_map.example.json new file mode 100644 index 0000000..797c8b6 --- /dev/null +++ b/exp_multimodal/alias_map.example.json @@ -0,0 +1,13 @@ +{ + "番茄": "西红柿", + "西红柿": "西红柿", + "胡瓜": "黄瓜", + "马铃薯": "土豆", + "洋芋": "土豆", + "芹菜(茎)": "芹菜", + "蒜薹": "蒜薹", + "蒜苔": "蒜薹", + "辣椒(青,尖)": "辣椒", + "辣椒(青，尖)": "辣椒" +} + diff --git a/exp_multimodal/exp_multimodal2.py b/exp_multimodal/exp_multimodal2.py new file mode 100644 index 0000000..3d58908 --- /dev/null +++ b/exp_multimodal/exp_multimodal2.py @@ -0,0 +1,63 @@ +import argparse +import json +import os +import sys +from typing import Dict + +# 兼容：支持直接运行脚本或用 -m 模块方式运行 +if __name__ == "__main__" and __package__ is None: + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +# 统一使用绝对导入，避免相对导入在脚本直跑时失败 +from exp_multimodal.labels import build_labels +from exp_multimodal.vlm_classifier import classify_image +from exp_multimodal.ollama_client import OLLAMA_URL, DEFAULT_MODEL + + +FEWSHOT_HINTS: Dict[str, str] = { + # 可选：仅对菜品模式提供少量文字提示，帮助区分相似菜 + # "西红柿鸡蛋": "红黄对比明显，蛋块与番茄块同炒，汤汁偏红", + # "麻婆豆腐": "豆腐块+红油辣椒+花椒，肉末点缀", +} + + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--mode", choices=["dish", "whole", "processed"], default="dish") + ap.add_argument("--image", default=r"D:\MyProjects\PythonProjects\FoodClassifier\dataset\DishClassification\test\回锅肉\img01.png") + ap.add_argument("--alias_map", default=None) + + args = ap.parse_args() + + print( + f"[Main] mode={args.mode} image={args.image} alias_map={args.alias_map} " + f"env_OLLAMA_URL={os.getenv('OLLAMA_URL')} env_VLM_MODEL={os.getenv('VLM_MODEL')} " + f"defaults url={OLLAMA_URL} model={DEFAULT_MODEL}" + ) + print(f"[Main] Image exists={os.path.exists(args.image)} size={os.path.getsize(args.image) if os.path.exists(args.image) else 'N/A'}") + + labels = build_labels(args.mode, args.alias_map) + ingredient_only = args.mode in {"whole", "processed"} + fewshot = FEWSHOT_HINTS if args.mode == "dish" else None + print(f"[Main] Built labels count={len(labels)} ingredient_only={ingredient_only} fewshot={bool(fewshot)}") + + res = classify_image( + args.image, + labels, + fewshot_hints=fewshot, + ingredient_only=ingredient_only, + ) + + print( + json.dumps( + {"mode": args.mode, "result": res, "num_labels": len(labels)}, + ensure_ascii=False, + ) + ) + + +if __name__ == "__main__": + main() + + diff --git a/exp_multimodal/labels.py b/exp_multimodal/labels.py new file mode 100644 index 0000000..7af500d --- /dev/null +++ b/exp_multimodal/labels.py @@ -0,0 +1,94 @@ +import json +import os +import re +from typing import Dict, List + + +def _list_dirs(path: str) -> List[str]: + if not os.path.isdir(path): + return [] + return [d for d in os.listdir(path) if os.path.isdir(os.path.join(path, d))] + + +def _normalize(name: str) -> str: + # 去括号/方括号内注释，统一去空格 + name = re.sub(r"[（(].*?[)）]", "", name) + name = re.sub(r"\[.*?\]", "", name) + return name.replace(" ", "").strip() + + +_SHAPE_COOKING_TERMS = [ + "切丝", + "切片", + "切丁", + "切段", + "切块", + "切条", + "切末", + "切花", + "丝", + "片", + "丁", + "段", + "块", + "条", + "末", + "清炒", + "红烧", + "爆炒", + "炒", + "煮", + "炖", + "蒸", + "炸", + "拌", + "焯", + "烤", + "煎", + "烩", + "烫", +] + + +def _base_ingredient(name: str) -> str: + n = _normalize(name) + for kw in _SHAPE_COOKING_TERMS: + n = n.replace(kw, "") + n = re.sub(r"[、，,·\-/]", "", n) + return n.strip() + + +def load_alias_map(fp: str | None = None) -> Dict[str, str]: + if fp and os.path.isfile(fp): + with open(fp, "r", encoding="utf-8") as f: + return json.load(f) + return {} + + +def build_labels(mode: str, alias_map_path: str | None = None) -> List[str]: + # 使 dataset 路径与运行目录无关：基于项目根目录（exp_multimodal 的上一级） + base = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "dataset") + if mode == "dish": + root = os.path.join(base, "DishClassification", "train") + elif mode == "whole": + root = os.path.join(base, "WholeIngredientRecognition", "train") + elif mode == "processed": + root = os.path.join(base, "ProcessedIngredientRecognition", "train") + else: + return [] + + raw = _list_dirs(root) + alias = load_alias_map(alias_map_path) + std: List[str] = [] + for r in raw: + n = _normalize(r) + n = alias.get(n, n) + if mode in ("whole", "processed"): + n = _base_ingredient(n) + if n: + std.append(n) + + # 去重并排序 + return sorted(list(dict.fromkeys(std))) + + diff --git a/exp_multimodal/ollama_client.py b/exp_multimodal/ollama_client.py new file mode 100644 index 0000000..3bb02e5 --- /dev/null +++ b/exp_multimodal/ollama_client.py @@ -0,0 +1,73 @@ +import base64 +import os +import time +from typing import Any, Dict, List + +import requests + + +OLLAMA_URL = os.getenv("OLLAMA_URL", "http://192.168.1.250:11434") +DEFAULT_MODEL = os.getenv("VLM_MODEL", "qwen2.5vl:32b") + + +def _img_to_data_uri(path: str) -> str: + with open(path, "rb") as f: + b64 = base64.b64encode(f.read()).decode("utf-8") + return f"data:image/jpeg;base64,{b64}" + + +def chat_vision( + prompt: str, + image_paths: List[str], + *, + temperature: float = 0.1, + options: Dict[str, Any] | None = None, + model: str = DEFAULT_MODEL, + timeout_sec: int = 120, +) -> str: + # Logging: request summary before heavy work + try: + img_sizes = [] + for p in image_paths: + try: + img_sizes.append(os.path.getsize(p)) + except Exception: + img_sizes.append(-1) + print( + f"[Ollama] Prepare chat url={OLLAMA_URL}/api/chat model={model} temp={temperature} " + f"images={len(image_paths)} sizes={img_sizes} timeout={timeout_sec}s" + ) + except Exception: + pass + + images = [_img_to_data_uri(p) for p in image_paths] + payload: Dict[str, Any] = { + "model": model, + "messages": [ + { + "role": "user", + "content": prompt, + "images": images, + } + ], + "options": {"temperature": temperature, **(options or {})}, + "stream": False, + } + + t0 = time.time() + try: + resp = requests.post(f"{OLLAMA_URL}/api/chat", json=payload, timeout=timeout_sec) + dt = time.time() - t0 + print(f"[Ollama] Response status={resp.status_code} elapsed={dt:.2f}s") + resp.raise_for_status() + data = resp.json() + text = (data.get("message") or {}).get("content", "") + _preview = (text[:120]).replace("\n", " ") + print(f"[Ollama] Text length={len(text)} preview={_preview}...") + return text + except Exception as e: + dt = time.time() - t0 + print(f"[Ollama][Error] during request elapsed={dt:.2f}s: {e}") + raise + + diff --git a/exp_multimodal/prompts.py b/exp_multimodal/prompts.py new file mode 100644 index 0000000..adaaf84 --- /dev/null +++ b/exp_multimodal/prompts.py @@ -0,0 +1,34 @@ +from typing import Dict, List, Optional + + +def build_closedset_prompt( + labels: List[str], + fewshot_hints: Optional[Dict[str, str]] = None, + ingredient_only: bool = False, +) -> str: + label_str = "，".join(labels) + + hint_lines: List[str] = [] + if fewshot_hints: + hint_lines.append("以下是少量文字示例帮助你区分：") + for k, v in list(fewshot_hints.items())[:20]: + hint_lines.append(f"- {k}：{v}") + hints = "\n".join(hint_lines) + + constraint = ( + "你只需输出食材名称，不要包含任何烹饪方式或形态描述（如切丝/切片/炒/煮/红烧/清炒等）。\n" + if ingredient_only + else "" + ) + + return ( + "你是食品图像分类专家。请严格在给定的标签集合中选择最符合图片内容的一个标签。\n" + "只允许返回一个标签，禁止输出集合外的词。请同时给出0-1置信度。\n" + f"{constraint}" + f"标签集合（封闭集，中文）：{label_str}\n" + f"{hints}\n" + '请仅输出合法 JSON：{"label":"<标签>","confidence":0-1}\n' + "不要输出任何额外解释或文本。" + ) + + diff --git a/exp_multimodal/vlm_classifier.py b/exp_multimodal/vlm_classifier.py new file mode 100644 index 0000000..12de74b --- /dev/null +++ b/exp_multimodal/vlm_classifier.py @@ -0,0 +1,64 @@ +import json +from typing import Dict, List, Optional + +from .ollama_client import chat_vision +from .prompts import build_closedset_prompt + + +def classify_image( + image_path: str, + labels: List[str], + fewshot_hints: Optional[Dict[str, str]] = None, + ingredient_only: bool = False, +) -> Dict: + if not labels: + print("[VLM] Empty labels provided, return Unknown") + return {"label": "Unknown", "confidence": 0.0} + + print( + f"[VLM] Start classify image={image_path} labels={len(labels)} " + f"ingredient_only={ingredient_only} fewshot={bool(fewshot_hints)}" + ) + prompt = build_closedset_prompt(labels, fewshot_hints, ingredient_only) + _preview_prompt = (prompt[:120]).replace("\n", " ") + print(f"[VLM] Prompt length={len(prompt)} preview={_preview_prompt}...") + + print("[VLM] Calling chat_vision...") + text = chat_vision(prompt, [image_path], temperature=0.1) + _preview_text = (text[:120]).replace("\n", " ") + print(f"[VLM] Received text length={len(text)} preview={_preview_text}...") + + # 主路径：严格 JSON 解析 + try: + obj = json.loads(text) + if isinstance(obj, dict) and "label" in obj: + label = obj.get("label", "") + # 纠偏：若输出不在封闭集，尝试包含匹配；否则 Unknown + if label not in labels: + for lb in labels: + if lb in str(label): + label = lb + break + else: + label = "Unknown" + conf = obj.get("confidence", 0.0) + try: + conf = float(conf) + except Exception: + conf = 0.0 + result = {"label": label, "confidence": conf} + print(f"[VLM] Parsed JSON result={result}") + return result + except Exception as e: + print(f"[VLM] JSON parse failed: {e}") + + # 回退：字符串包含匹配 + for lb in labels: + if lb in text: + print(f"[VLM] Fallback match label={lb}") + return {"label": lb, "confidence": 0.5} + + print("[VLM] No match, return Unknown") + return {"label": "Unknown", "confidence": 0.0} + +