canteen/StallDualScreen
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

功能优化

Browse Source
This commit is contained in:
mazengfei
2025-11-17 16:09:01 +08:00
parent b4087e435e
commit cb9d68d2e3
14 changed files with 233 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/src/main/java/com/sw/dualscreen/activity/FoodCollectionActivity.kt
+11 -7
View File
@@ -48,9 +48,12 @@ import org.pytorch.IValue
import org.pytorch.Module
import org.pytorch.torchvision.TensorImageUtils
import timber.log.Timber
import kotlin.math.max
class FoodCollectionActivity : BaseActivity<ActivityFoodCollectionBinding>() {
companion object {
val MAX_COUNT = 100
}
private val viewModel by viewModels<UserViewModel>()
private var selectedFoodId: String? = ""
private var selectedFoodName: String? = ""
@@ -64,6 +67,7 @@ class FoodCollectionActivity : BaseActivity<ActivityFoodCollectionBinding>() {
private val foodList = mutableListOf<FoodInfo>() // 适配器内部维护的数据列表
private val debouncer = Debouncer(2000)
private lateinit var previewView: PreviewView
private val cameraUtils: CameraUtils by lazy {
CameraUtils(this)
}
@@ -101,7 +105,7 @@ class FoodCollectionActivity : BaseActivity<ActivityFoodCollectionBinding>() {
private val cameraCallback: (Uri) -> Unit = { uri ->
val index = foodCollectionList.indexOfFirst { it.bitmap == null }
if (index == -1) {
ToastUtils.showToast("每次只允许保存6条数据")
ToastUtils.showToast("每次只允许保存${MAX_COUNT}条数据")
rerurn@ cameraCallback
}
ImageUtil.uriToBitmap(this, uri)?.let { bitmap ->
@@ -127,8 +131,8 @@ class FoodCollectionActivity : BaseActivity<ActivityFoodCollectionBinding>() {
@SuppressLint("NotifyDataSetChanged")
private fun takePhoto() {
val count = foodCollectionList.count { it.bitmap != null }
if (count == 6) {
ToastUtils.showToast("每次只允许保存6条数据")
if (count >= MAX_COUNT) {
ToastUtils.showToast("每次只允许保存${MAX_COUNT}条数据")
return
}
cameraUtils.takePhoto(cameraCallback)
@@ -166,7 +170,7 @@ class FoodCollectionActivity : BaseActivity<ActivityFoodCollectionBinding>() {
startActivity(intent)
finish()
}
repeat(6) {
repeat(MAX_COUNT) {
foodCollectionList.add(FoodCollectionBean(isShowCamera = true))
}
binding.rvFoodCollection.let {
@@ -208,8 +212,8 @@ class FoodCollectionActivity : BaseActivity<ActivityFoodCollectionBinding>() {
binding.btnTakePhoto.clickWithDebounce {
binding.btnTakePhoto.text = "拍照"
val count = foodCollectionList.count { it.bitmap != null }
if (count == 6) {
ToastUtils.showToast("每次只允许保存6条数据")
if (count >= MAX_COUNT) {
ToastUtils.showToast("每次只允许保存${MAX_COUNT}条数据")
return@clickWithDebounce
}
// cameraHelper.openCamera()
app/src/main/java/com/sw/dualscreen/activity/MainActivity.kt
+82 -39
View File
@@ -32,8 +32,10 @@ import com.sw.dualscreen.adapter.dpToPx
import com.sw.dualscreen.databinding.ActivityMainBinding
import com.sw.dualscreen.databinding.ItemFoodInfoBinding
import com.sw.dualscreen.ext.dp
import com.sw.dualscreen.ext.format2String
import com.sw.dualscreen.model.response.FoodInfo
import com.sw.dualscreen.objbox.FoodModule
import com.sw.dualscreen.objbox.FoodModule.IdNameScore
import com.sw.dualscreen.presentation.SecondaryScreenPresentation
import com.sw.dualscreen.sdk.SensorScaleUtils
import com.sw.dualscreen.utils.AssetsTool
@@ -54,6 +56,7 @@ import kotlinx.coroutines.launch
import timber.log.Timber
import java.io.File
import java.util.concurrent.Executors
import kotlin.math.roundToInt
/**
* 服务员显示界面
@@ -138,6 +141,8 @@ class MainActivity : BaseActivity<ActivityMainBinding>() {
updateCurrentFood(null)
isRecognitionFood = true
presentation?.updateMealPickupMode(position)
SensorScaleUtils.zero()
}
override fun onNothingSelected(parent: AdapterView<*>?) {}
@@ -156,6 +161,9 @@ class MainActivity : BaseActivity<ActivityMainBinding>() {
binding.tvFoodName.setOnClickListener {
SensorScaleUtils.zero()
}
binding.tvRescan.setOnClickListener {
debouncer.debounce { recognizeFood() }
}
}
fun updateCurrentFood(foodInfo: FoodInfo?) {
@@ -189,19 +197,11 @@ class MainActivity : BaseActivity<ActivityMainBinding>() {
override fun registerDataChange() {
super.registerDataChange()
lifecycleScope.launch {
viewModel.identifiedFoodInfoList2.collect { list ->
Timber.d("registerDataChange识别后查询接口数据：${GsonUtils.toJson(list)}")
adapter.updateData(list)
if (list.isNotEmpty()) {
checkedItem = list[0]
checkedItem!!.photoUri = lastPhotoUri
updateCurrentFood(checkedItem)
lastPhotoUri = null
LightManager.closeRedLight()
}
}
}
// lifecycleScope.launch {
// viewModel.identifiedFoodInfoList2.collect { list ->
// updateFoodInfo(list)
// }
// }
lifecycleScope.launch {
viewModel.identifiedFoodInfoList.collect { list ->
adapter.updateData(list)
@@ -217,7 +217,7 @@ class MainActivity : BaseActivity<ActivityMainBinding>() {
Timber.d("registerDataChange weight = $weight")
presentation?.updateWeight(weight)
if (weight <= 0.005 && presentation?.mealPickupMode == 1) {//余量取餐，检测到秤上没有东西，重新启动识别菜品
if (weight <= 0.005) {//余量取餐，检测到秤上没有东西，重新启动识别菜品 && presentation?.mealPickupMode == 1
isRecognitionFood = true
}
@@ -226,42 +226,85 @@ class MainActivity : BaseActivity<ActivityMainBinding>() {
isRecognitionFood = false
}
debouncer.debounce {
LightManager.openRedLight()
LightManager.openGreenLight()
takePhoto { photoUri ->
lastPhotoUri = photoUri
Timber.d("registerDataChange photoUri = ${photoUri.path}")
// viewModel.getIdentifiedFoodList()
ImageUtil.uriToBitmap(this, photoUri)?.let { bitmap ->
Timber.d("registerDataChange photoUri 拿到bitmap")
// val bmp = BitmapCropper.cropCenter(bitmap, 1300, 900)
Timber.d("registerDataChange photoUri bitmap裁剪完成")
val file = BitmapSaver.saveToAppFilesDir(
bitmap,
this,
"IMG_CROP_${System.currentTimeMillis()}.jpg"
)
Timber.d("registerDataChange photoUri bitmap保存文件路径：${file?.absolutePath}")
val resultList = FoodModule.queryFood(bitmap)
Timber.d("registerDataChange photoUri 拿到识别数据")
val foodName = resultList.joinToString(separator = ",")
Timber.d("registerDataChange photoUri 识别数据名称:$foodName")
viewModel.getFoodInfo(foodName)
}
}
recognizeFood()
}
}
lastWeight = weight
}
}
private fun recognizeFood() {
LightManager.openRedLight()
LightManager.openGreenLight()
takePhoto { photoUri ->
lastPhotoUri = photoUri
Timber.d("registerDataChange photoUri = ${photoUri.path}")
// viewModel.getIdentifiedFoodList()
ImageUtil.uriToBitmap(this, photoUri)?.let { bitmap ->
Timber.d("registerDataChange photoUri 拿到bitmap")
// val bmp = BitmapCropper.cropCenter(bitmap, 1300, 900)
Timber.d("registerDataChange photoUri bitmap裁剪完成")
val file = BitmapSaver.saveToAppFilesDir(
bitmap,
this,
"IMG_CROP_${System.currentTimeMillis()}.jpg"
)
Timber.d("registerDataChange photoUri bitmap保存文件路径：${file?.absolutePath}")
//val nameList = FoodModule.queryFood(bitmap)
val scoreList = FoodModule.getFoodScoreList(bitmap)
Timber.d(
"registerDataChange photoUri 拿到识别数据:${
GsonUtils.toJson(
scoreList
)
}"
)
val nameList = scoreList.map { it.name }
val foodName = nameList.joinToString(separator = ",")
Timber.d("registerDataChange photoUri 识别数据名称:$foodName")
if (TextUtils.isEmpty(foodName)) {
LightManager.closeGreenLight()
LightManager.closeRedLight()
binding.layoutRescan.visibility = View.VISIBLE
} else {
binding.layoutRescan.visibility = View.GONE
viewModel.getFoodInfo(foodName) { list ->
runOnUiThread {
updateFoodInfo(list, scoreList)
}
}
}
}
}
}
private fun updateFoodInfo(list: List<FoodInfo>, scoreList: List<IdNameScore>) {
Timber.d("registerDataChange识别后查询接口数据：${GsonUtils.toJson(list)}")
list.forEach { foodInfo ->
val scoreItem = scoreList.firstOrNull { it.name == foodInfo.foodName }
val score = scoreItem?.score ?: 0.0
foodInfo.score = ((1 - score) * 10000).roundToInt()
}
adapter.updateData(list)
if (list.isNotEmpty()) {
checkedItem = list[0]
checkedItem!!.photoUri = lastPhotoUri
updateCurrentFood(checkedItem)
lastPhotoUri = null
LightManager.closeRedLight()
}
}
private fun createAdapter(): GenericItemAdapter<FoodInfo, ItemFoodInfoBinding> {
return GenericItemAdapter(
items = emptyList(),
bindingInflater = ItemFoodInfoBinding::inflate,
bindCallback = { item, position ->
this.tvName.text = item.foodName
val score = (item.score / 100.0).format2String(2)
this.tvName.text = "${item.foodName}\n${score}%"
if (item.id != checkedItem?.id) {
this.tvName.typeface = Typeface.defaultFromStyle(Typeface.NORMAL)
this.tvName.setTextColor(resources.getColor(R.color.identify_normal))
app/src/main/java/com/sw/dualscreen/ext/CommonExt.kt
+1
View File
@@ -66,6 +66,7 @@ val Float.sp: Float
)
fun Double.format2String(): String = "%.1f".format(this)
fun Double.format2String(num:Int): String = "%.${num}f".format(this)
// 添加扩展函数
fun ImageProxy.toSafeBitmap(): Bitmap {
app/src/main/java/com/sw/dualscreen/model/response/FoodInfo.kt
+2 -1
View File
@@ -38,7 +38,8 @@ data class FoodInfo(
@SerializedName("stFoodInfoSetting")
val stFoodInfoSetting: StFoodInfoSetting? = StFoodInfoSetting(),
@SerializedName("stFoodInfoSpecificationList")
val stFoodInfoSpecificationList: List<StFoodInfoSpecification?>? = listOf()
val stFoodInfoSpecificationList: List<StFoodInfoSpecification?>? = listOf(),
var score: Int = 0
) : Parcelable {
@Parcelize
data class FoodTypeAndRealIntakeVo(
app/src/main/java/com/sw/dualscreen/objbox/FoodModule.kt
+73 -27
View File
@@ -1,12 +1,14 @@
package com.sw.dualscreen.objbox
import com.sw.dualscreen.MyApp
import com.sw.dualscreen.utils.AssetsTool
import android.content.Context
import android.graphics.Bitmap
import android.net.Uri
import android.util.SparseLongArray
import com.google.gson.Gson
import com.google.gson.reflect.TypeToken
import com.sw.dualscreen.MyApp
import com.sw.dualscreen.utils.AssetsTool
import com.sw.dualscreen.utils.GsonUtils
import com.sw.dualscreen.utils.ImageUtil
import io.objectbox.Box
import io.objectbox.kotlin.boxFor
@@ -22,6 +24,7 @@ import java.io.InputStream
object FoodModule {
private const val THRESHOLD = 0.8
private lateinit var module_mobile: Module
private lateinit var box: Box<Food>
private lateinit var embeddingsList: List<List<Float>>
@@ -68,50 +71,93 @@ object FoodModule {
}
}
/**
* 返回识别物品名称列表
*/
fun queryFood(bitmap: Bitmap, queryCount: Int = 15): List<String> {
val inputTensor =
TensorImageUtils.bitmapToFloat32Tensor(
bitmap,
NO_MEAN_RGB, // [0.485, 0.456, 0.406] TORCHVISION_NORM_MEAN_RGB
NO_STD_RGB // [0.229, 0.224, 0.225] TORCHVISION_NORM_STD_RGB
)
val outputTensor = module_mobile.forward(IValue.from(inputTensor)).toTensor()
val floatArray = outputTensor.dataAsFloatArray
if (bitmap.isRecycled.not()) {
bitmap.recycle()
}
val floatArray = bitmap2FloatArray(bitmap)
return queryFood(floatArray, queryCount)
}
/**
* 返回识别物品IdNameScore对象列表
*/
fun queryFoodNameScore(bitmap: Bitmap, queryCount: Int = 15): List<IdNameScore> {
val floatArray = bitmap2FloatArray(bitmap)
return queryFoodNameScore(floatArray, queryCount)
}
fun queryFoodNameScore(floatArray: FloatArray, queryCount: Int = 15): List<IdNameScore> {
val query: Query<Food> =
box.query(Food_.foodVector.nearestNeighbors(floatArray, queryCount)).build()
//查询比较分数
// val tempList = query.findWithScores().sortedBy { it.score }.map { "${it.get().name}|${it.get().foodIdx}|${it.score}" }
val idScoreList = query.findIdsWithScores()
val nameScoreList = mutableListOf<IdNameScore>()
idScoreList.forEach {
nameScoreList.add(IdNameScore(id = it.id, name = box.get(it.id).name?:"", score = it.score))
}
Timber.tag("FoodModule").d("registerDataChange,queryFood数据：${GsonUtils.toJson(nameScoreList)}")
return nameScoreList
}
fun getFoodScoreList(bitmap: Bitmap, queryCount: Int = 15): List<IdNameScore> {
val floatArray = bitmap2FloatArray(bitmap)
val nameScoreList = queryFoodNameScore(floatArray, queryCount)
val maxScoreList = nameScoreList
.filter { it.score < 1 - THRESHOLD }
.groupBy { it.name }
.map { (_, value) -> value.minByOrNull { it.score }!! }
.toMutableList()
val map = mutableMapOf<String, Int>()
nameScoreList.forEach {
val key = it.name
val count = map[key] ?: 0
map[key] = count + 1
}
val orderList = map.entries.sortedByDescending { it.value }.map { it.key }.toMutableList()
val firstFood = nameScoreList[0].name
orderList.remove(firstFood)
orderList.add(0, firstFood)
val sortedScoreList = maxScoreList.sortedWith(compareBy {
orderList.indexOf(it.name)
})
//Timber.tag("FoodModule").d("getFoodScoreList数据：${sortedScoreList.toJsonString()}")
return sortedScoreList
}
fun queryFood(floatArray: FloatArray, queryCount: Int = 15): List<String> {
val query: Query<Food> =
box.query(Food_.foodVector.nearestNeighbors(floatArray, queryCount)).build()
//查询比较分数
// val tempList = query.findWithScores().sortedBy { it.score }.map { "${it.get().name}|${it.get().foodIdx}|${it.score}" }
val map = mutableMapOf<String, Int>()
val idScoreList = query.findIdsWithScores()
val nameScoreList = mutableListOf<IdNameScore>()
idScoreList.forEach {
nameScoreList.add(IdNameScore(id = it.id, name = box.get(it.id).name?:"", score = it.score))
Timber.tag("registerDataChange").d("queryFood数据：${box.get(it.id).name}===score=${it.score}")
}
// Timber.tag("FoodModule").d("queryFood数据：${nameScoreList.toJsonString()}")
idScoreList.filter { it.score < 0.20 }.forEach {
query.findIdsWithScores().forEach {
val food = box.get(it.id)
//FoodQueryResult(id = it.id, name = food.name, foodIdx = food.foodIdx, score = it.score)
food.name?.let { key ->
val count = map[key] ?: 0
map[key] = count + 1
Timber.d("${food.name}|${food.foodIdx}|${it.score}")
if (1 - it.score >= THRESHOLD) {
//FoodQueryResult(id = it.id, name = food.name, foodIdx = food.foodIdx, score = it.score)
food.name?.let { key ->
val count = map[key] ?: 0
map.put(key, count + 1)
}
}
}
val list = map.entries.sortedByDescending { it.value }.map { it.key }
return list
// val map = mutableMapOf<String, Int>()
// val nameScoreList = queryFoodNameScore(floatArray, queryCount)
// nameScoreList.filter { it.score < 0.05 }.forEach {
// val count = map[it.name] ?: 0
// map[it.name] = count + 1
// }
// val list = map.entries.sortedByDescending { it.value }.map { it.key }
// return list
}
data class IdNameScore(
val id:Long,
val name:String,
var name:String,
val score: Double
)
app/src/main/java/com/sw/dualscreen/utils/BitmapSaver.kt
+4 -1
View File
@@ -30,7 +30,10 @@ object BitmapSaver {
quality: Int = 100
): File? {
//val dir = context.getExternalFilesDir(null)
val dir = context.cacheDir
val dir = File(context.cacheDir, "crop")
if (dir.exists().not()) {
dir.mkdirs()
}
return saveBitmap(bitmap, File(dir, fileName), format, quality)
}
app/src/main/java/com/sw/dualscreen/viewmodel/UserViewModel.kt
+5 -2
View File
@@ -23,6 +23,7 @@ import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import timber.log.Timber
import kotlin.math.atan
class UserViewModel : BaseViewModel() {
@@ -210,12 +211,14 @@ class UserViewModel : BaseViewModel() {
}
}
fun getFoodInfo(foodName: String) {
fun getFoodInfo(foodName: String, action:(List<FoodInfo>)-> Unit = {}) {
Timber.d("getFoodInfo")
launchWithLoading {
val response = repository.getFoodInfo(foodName = foodName)
if (parseResponse(response)) {
_identifiedFoodInfoList2.value = response.result ?: emptyList()
val list = response.result ?: emptyList()
_identifiedFoodInfoList2.value = list
action(list)
}
}
}