canteen/StallDualScreen
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

功能优化

Browse Source
This commit is contained in:
mazengfei
2025-11-17 16:09:01 +08:00
parent b4087e435e
commit cb9d68d2e3
14 changed files with 233 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.idea/compiler.xml
Generated
+6
View File
@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CompilerConfiguration">
<bytecodeTargetLevel target="17" />
</component>
</project>
.idea/deploymentTargetSelector.xml
Generated
+2 -2
View File
@@ -4,10 +4,10 @@
<selectionStates>
<SelectionState runConfigName="app">
<option name="selectionMode" value="DROPDOWN" />
<DropdownSelection timestamp="2025-10-16T11:00:08.044723900Z">
<DropdownSelection timestamp="2025-11-07T10:55:54.677927600Z">
<Target type="DEFAULT_BOOT">
<handle>
<DeviceId pluginId="Default" identifier="serial=192.168.1.233:5555;connection=5022ef29" />
<DeviceId pluginId="Default" identifier="serial=192.168.1.56:5555;connection=2687c52a" />
</handle>
</Target>
</DropdownSelection>
.idea/gradle.xml
Generated
+1 -1
View File
@@ -6,7 +6,7 @@
<GradleProjectSettings>
<option name="testRunner" value="CHOOSE_PER_TEST" />
<option name="externalProjectPath" value="$PROJECT_DIR$" />
<option name="gradleJvm" value="zulu-17" />
<option name="gradleJvm" value="azul-17" />
<option name="modules">
<set>
<option value="$PROJECT_DIR$" />
app/src/main/java/com/sw/dualscreen/activity/FoodCollectionActivity.kt
+11 -7
View File
@@ -48,9 +48,12 @@ import org.pytorch.IValue
import org.pytorch.Module
import org.pytorch.torchvision.TensorImageUtils
import timber.log.Timber
import kotlin.math.max
class FoodCollectionActivity : BaseActivity<ActivityFoodCollectionBinding>() {
companion object {
val MAX_COUNT = 100
}
private val viewModel by viewModels<UserViewModel>()
private var selectedFoodId: String? = ""
private var selectedFoodName: String? = ""
@@ -64,6 +67,7 @@ class FoodCollectionActivity : BaseActivity<ActivityFoodCollectionBinding>() {
private val foodList = mutableListOf<FoodInfo>() // 适配器内部维护的数据列表
private val debouncer = Debouncer(2000)
private lateinit var previewView: PreviewView
private val cameraUtils: CameraUtils by lazy {
CameraUtils(this)
}
@@ -101,7 +105,7 @@ class FoodCollectionActivity : BaseActivity<ActivityFoodCollectionBinding>() {
private val cameraCallback: (Uri) -> Unit = { uri ->
val index = foodCollectionList.indexOfFirst { it.bitmap == null }
if (index == -1) {
ToastUtils.showToast("每次只允许保存6条数据")
ToastUtils.showToast("每次只允许保存${MAX_COUNT}条数据")
rerurn@ cameraCallback
}
ImageUtil.uriToBitmap(this, uri)?.let { bitmap ->
@@ -127,8 +131,8 @@ class FoodCollectionActivity : BaseActivity<ActivityFoodCollectionBinding>() {
@SuppressLint("NotifyDataSetChanged")
private fun takePhoto() {
val count = foodCollectionList.count { it.bitmap != null }
if (count == 6) {
ToastUtils.showToast("每次只允许保存6条数据")
if (count >= MAX_COUNT) {
ToastUtils.showToast("每次只允许保存${MAX_COUNT}条数据")
return
}
cameraUtils.takePhoto(cameraCallback)
@@ -166,7 +170,7 @@ class FoodCollectionActivity : BaseActivity<ActivityFoodCollectionBinding>() {
startActivity(intent)
finish()
}
repeat(6) {
repeat(MAX_COUNT) {
foodCollectionList.add(FoodCollectionBean(isShowCamera = true))
}
binding.rvFoodCollection.let {
@@ -208,8 +212,8 @@ class FoodCollectionActivity : BaseActivity<ActivityFoodCollectionBinding>() {
binding.btnTakePhoto.clickWithDebounce {
binding.btnTakePhoto.text = "拍照"
val count = foodCollectionList.count { it.bitmap != null }
if (count == 6) {
ToastUtils.showToast("每次只允许保存6条数据")
if (count >= MAX_COUNT) {
ToastUtils.showToast("每次只允许保存${MAX_COUNT}条数据")
return@clickWithDebounce
}
// cameraHelper.openCamera()
app/src/main/java/com/sw/dualscreen/activity/MainActivity.kt
+82 -39
View File
@@ -32,8 +32,10 @@ import com.sw.dualscreen.adapter.dpToPx
import com.sw.dualscreen.databinding.ActivityMainBinding
import com.sw.dualscreen.databinding.ItemFoodInfoBinding
import com.sw.dualscreen.ext.dp
import com.sw.dualscreen.ext.format2String
import com.sw.dualscreen.model.response.FoodInfo
import com.sw.dualscreen.objbox.FoodModule
import com.sw.dualscreen.objbox.FoodModule.IdNameScore
import com.sw.dualscreen.presentation.SecondaryScreenPresentation
import com.sw.dualscreen.sdk.SensorScaleUtils
import com.sw.dualscreen.utils.AssetsTool
@@ -54,6 +56,7 @@ import kotlinx.coroutines.launch
import timber.log.Timber
import java.io.File
import java.util.concurrent.Executors
import kotlin.math.roundToInt
/**
* 服务员显示界面
@@ -138,6 +141,8 @@ class MainActivity : BaseActivity<ActivityMainBinding>() {
updateCurrentFood(null)
isRecognitionFood = true
presentation?.updateMealPickupMode(position)
SensorScaleUtils.zero()
}
override fun onNothingSelected(parent: AdapterView<*>?) {}
@@ -156,6 +161,9 @@ class MainActivity : BaseActivity<ActivityMainBinding>() {
binding.tvFoodName.setOnClickListener {
SensorScaleUtils.zero()
}
binding.tvRescan.setOnClickListener {
debouncer.debounce { recognizeFood() }
}
}
fun updateCurrentFood(foodInfo: FoodInfo?) {
@@ -189,19 +197,11 @@ class MainActivity : BaseActivity<ActivityMainBinding>() {
override fun registerDataChange() {
super.registerDataChange()
lifecycleScope.launch {
viewModel.identifiedFoodInfoList2.collect { list ->
Timber.d("registerDataChange识别后查询接口数据：${GsonUtils.toJson(list)}")
adapter.updateData(list)
if (list.isNotEmpty()) {
checkedItem = list[0]
checkedItem!!.photoUri = lastPhotoUri
updateCurrentFood(checkedItem)
lastPhotoUri = null
LightManager.closeRedLight()
}
}
}
// lifecycleScope.launch {
// viewModel.identifiedFoodInfoList2.collect { list ->
// updateFoodInfo(list)
// }
// }
lifecycleScope.launch {
viewModel.identifiedFoodInfoList.collect { list ->
adapter.updateData(list)
@@ -217,7 +217,7 @@ class MainActivity : BaseActivity<ActivityMainBinding>() {
Timber.d("registerDataChange weight = $weight")
presentation?.updateWeight(weight)
if (weight <= 0.005 && presentation?.mealPickupMode == 1) {//余量取餐，检测到秤上没有东西，重新启动识别菜品
if (weight <= 0.005) {//余量取餐，检测到秤上没有东西，重新启动识别菜品 && presentation?.mealPickupMode == 1
isRecognitionFood = true
}
@@ -226,42 +226,85 @@ class MainActivity : BaseActivity<ActivityMainBinding>() {
isRecognitionFood = false
}
debouncer.debounce {
LightManager.openRedLight()
LightManager.openGreenLight()
takePhoto { photoUri ->
lastPhotoUri = photoUri
Timber.d("registerDataChange photoUri = ${photoUri.path}")
// viewModel.getIdentifiedFoodList()
ImageUtil.uriToBitmap(this, photoUri)?.let { bitmap ->
Timber.d("registerDataChange photoUri 拿到bitmap")
// val bmp = BitmapCropper.cropCenter(bitmap, 1300, 900)
Timber.d("registerDataChange photoUri bitmap裁剪完成")
val file = BitmapSaver.saveToAppFilesDir(
bitmap,
this,
"IMG_CROP_${System.currentTimeMillis()}.jpg"
)
Timber.d("registerDataChange photoUri bitmap保存文件路径：${file?.absolutePath}")
val resultList = FoodModule.queryFood(bitmap)
Timber.d("registerDataChange photoUri 拿到识别数据")
val foodName = resultList.joinToString(separator = ",")
Timber.d("registerDataChange photoUri 识别数据名称:$foodName")
viewModel.getFoodInfo(foodName)
}
}
recognizeFood()
}
}
lastWeight = weight
}
}
private fun recognizeFood() {
LightManager.openRedLight()
LightManager.openGreenLight()
takePhoto { photoUri ->
lastPhotoUri = photoUri
Timber.d("registerDataChange photoUri = ${photoUri.path}")
// viewModel.getIdentifiedFoodList()
ImageUtil.uriToBitmap(this, photoUri)?.let { bitmap ->
Timber.d("registerDataChange photoUri 拿到bitmap")
// val bmp = BitmapCropper.cropCenter(bitmap, 1300, 900)
Timber.d("registerDataChange photoUri bitmap裁剪完成")
val file = BitmapSaver.saveToAppFilesDir(
bitmap,
this,
"IMG_CROP_${System.currentTimeMillis()}.jpg"
)
Timber.d("registerDataChange photoUri bitmap保存文件路径：${file?.absolutePath}")
//val nameList = FoodModule.queryFood(bitmap)
val scoreList = FoodModule.getFoodScoreList(bitmap)
Timber.d(
"registerDataChange photoUri 拿到识别数据:${
GsonUtils.toJson(
scoreList
)
}"
)
val nameList = scoreList.map { it.name }
val foodName = nameList.joinToString(separator = ",")
Timber.d("registerDataChange photoUri 识别数据名称:$foodName")
if (TextUtils.isEmpty(foodName)) {
LightManager.closeGreenLight()
LightManager.closeRedLight()
binding.layoutRescan.visibility = View.VISIBLE
} else {
binding.layoutRescan.visibility = View.GONE
viewModel.getFoodInfo(foodName) { list ->
runOnUiThread {
updateFoodInfo(list, scoreList)
}
}
}
}
}
}
private fun updateFoodInfo(list: List<FoodInfo>, scoreList: List<IdNameScore>) {
Timber.d("registerDataChange识别后查询接口数据：${GsonUtils.toJson(list)}")
list.forEach { foodInfo ->
val scoreItem = scoreList.firstOrNull { it.name == foodInfo.foodName }
val score = scoreItem?.score ?: 0.0
foodInfo.score = ((1 - score) * 10000).roundToInt()
}
adapter.updateData(list)
if (list.isNotEmpty()) {
checkedItem = list[0]
checkedItem!!.photoUri = lastPhotoUri
updateCurrentFood(checkedItem)
lastPhotoUri = null
LightManager.closeRedLight()
}
}
private fun createAdapter(): GenericItemAdapter<FoodInfo, ItemFoodInfoBinding> {
return GenericItemAdapter(
items = emptyList(),
bindingInflater = ItemFoodInfoBinding::inflate,
bindCallback = { item, position ->
this.tvName.text = item.foodName
val score = (item.score / 100.0).format2String(2)
this.tvName.text = "${item.foodName}\n${score}%"
if (item.id != checkedItem?.id) {
this.tvName.typeface = Typeface.defaultFromStyle(Typeface.NORMAL)
this.tvName.setTextColor(resources.getColor(R.color.identify_normal))
app/src/main/java/com/sw/dualscreen/ext/CommonExt.kt
+1
View File
@@ -66,6 +66,7 @@ val Float.sp: Float
)
fun Double.format2String(): String = "%.1f".format(this)
fun Double.format2String(num:Int): String = "%.${num}f".format(this)
// 添加扩展函数
fun ImageProxy.toSafeBitmap(): Bitmap {
app/src/main/java/com/sw/dualscreen/model/response/FoodInfo.kt
+2 -1
View File
@@ -38,7 +38,8 @@ data class FoodInfo(
@SerializedName("stFoodInfoSetting")
val stFoodInfoSetting: StFoodInfoSetting? = StFoodInfoSetting(),
@SerializedName("stFoodInfoSpecificationList")
val stFoodInfoSpecificationList: List<StFoodInfoSpecification?>? = listOf()
val stFoodInfoSpecificationList: List<StFoodInfoSpecification?>? = listOf(),
var score: Int = 0
) : Parcelable {
@Parcelize
data class FoodTypeAndRealIntakeVo(
app/src/main/java/com/sw/dualscreen/objbox/FoodModule.kt
+73 -27
View File
@@ -1,12 +1,14 @@
package com.sw.dualscreen.objbox
import com.sw.dualscreen.MyApp
import com.sw.dualscreen.utils.AssetsTool
import android.content.Context
import android.graphics.Bitmap
import android.net.Uri
import android.util.SparseLongArray
import com.google.gson.Gson
import com.google.gson.reflect.TypeToken
import com.sw.dualscreen.MyApp
import com.sw.dualscreen.utils.AssetsTool
import com.sw.dualscreen.utils.GsonUtils
import com.sw.dualscreen.utils.ImageUtil
import io.objectbox.Box
import io.objectbox.kotlin.boxFor
@@ -22,6 +24,7 @@ import java.io.InputStream
object FoodModule {
private const val THRESHOLD = 0.8
private lateinit var module_mobile: Module
private lateinit var box: Box<Food>
private lateinit var embeddingsList: List<List<Float>>
@@ -68,50 +71,93 @@ object FoodModule {
}
}
/**
* 返回识别物品名称列表
*/
fun queryFood(bitmap: Bitmap, queryCount: Int = 15): List<String> {
val inputTensor =
TensorImageUtils.bitmapToFloat32Tensor(
bitmap,
NO_MEAN_RGB, // [0.485, 0.456, 0.406] TORCHVISION_NORM_MEAN_RGB
NO_STD_RGB // [0.229, 0.224, 0.225] TORCHVISION_NORM_STD_RGB
)
val outputTensor = module_mobile.forward(IValue.from(inputTensor)).toTensor()
val floatArray = outputTensor.dataAsFloatArray
if (bitmap.isRecycled.not()) {
bitmap.recycle()
}
val floatArray = bitmap2FloatArray(bitmap)
return queryFood(floatArray, queryCount)
}
/**
* 返回识别物品IdNameScore对象列表
*/
fun queryFoodNameScore(bitmap: Bitmap, queryCount: Int = 15): List<IdNameScore> {
val floatArray = bitmap2FloatArray(bitmap)
return queryFoodNameScore(floatArray, queryCount)
}
fun queryFoodNameScore(floatArray: FloatArray, queryCount: Int = 15): List<IdNameScore> {
val query: Query<Food> =
box.query(Food_.foodVector.nearestNeighbors(floatArray, queryCount)).build()
//查询比较分数
// val tempList = query.findWithScores().sortedBy { it.score }.map { "${it.get().name}|${it.get().foodIdx}|${it.score}" }
val idScoreList = query.findIdsWithScores()
val nameScoreList = mutableListOf<IdNameScore>()
idScoreList.forEach {
nameScoreList.add(IdNameScore(id = it.id, name = box.get(it.id).name?:"", score = it.score))
}
Timber.tag("FoodModule").d("registerDataChange,queryFood数据：${GsonUtils.toJson(nameScoreList)}")
return nameScoreList
}
fun getFoodScoreList(bitmap: Bitmap, queryCount: Int = 15): List<IdNameScore> {
val floatArray = bitmap2FloatArray(bitmap)
val nameScoreList = queryFoodNameScore(floatArray, queryCount)
val maxScoreList = nameScoreList
.filter { it.score < 1 - THRESHOLD }
.groupBy { it.name }
.map { (_, value) -> value.minByOrNull { it.score }!! }
.toMutableList()
val map = mutableMapOf<String, Int>()
nameScoreList.forEach {
val key = it.name
val count = map[key] ?: 0
map[key] = count + 1
}
val orderList = map.entries.sortedByDescending { it.value }.map { it.key }.toMutableList()
val firstFood = nameScoreList[0].name
orderList.remove(firstFood)
orderList.add(0, firstFood)
val sortedScoreList = maxScoreList.sortedWith(compareBy {
orderList.indexOf(it.name)
})
//Timber.tag("FoodModule").d("getFoodScoreList数据：${sortedScoreList.toJsonString()}")
return sortedScoreList
}
fun queryFood(floatArray: FloatArray, queryCount: Int = 15): List<String> {
val query: Query<Food> =
box.query(Food_.foodVector.nearestNeighbors(floatArray, queryCount)).build()
//查询比较分数
// val tempList = query.findWithScores().sortedBy { it.score }.map { "${it.get().name}|${it.get().foodIdx}|${it.score}" }
val map = mutableMapOf<String, Int>()
val idScoreList = query.findIdsWithScores()
val nameScoreList = mutableListOf<IdNameScore>()
idScoreList.forEach {
nameScoreList.add(IdNameScore(id = it.id, name = box.get(it.id).name?:"", score = it.score))
Timber.tag("registerDataChange").d("queryFood数据：${box.get(it.id).name}===score=${it.score}")
}
// Timber.tag("FoodModule").d("queryFood数据：${nameScoreList.toJsonString()}")
idScoreList.filter { it.score < 0.20 }.forEach {
query.findIdsWithScores().forEach {
val food = box.get(it.id)
//FoodQueryResult(id = it.id, name = food.name, foodIdx = food.foodIdx, score = it.score)
food.name?.let { key ->
val count = map[key] ?: 0
map[key] = count + 1
Timber.d("${food.name}|${food.foodIdx}|${it.score}")
if (1 - it.score >= THRESHOLD) {
//FoodQueryResult(id = it.id, name = food.name, foodIdx = food.foodIdx, score = it.score)
food.name?.let { key ->
val count = map[key] ?: 0
map.put(key, count + 1)
}
}
}
val list = map.entries.sortedByDescending { it.value }.map { it.key }
return list
// val map = mutableMapOf<String, Int>()
// val nameScoreList = queryFoodNameScore(floatArray, queryCount)
// nameScoreList.filter { it.score < 0.05 }.forEach {
// val count = map[it.name] ?: 0
// map[it.name] = count + 1
// }
// val list = map.entries.sortedByDescending { it.value }.map { it.key }
// return list
}
data class IdNameScore(
val id:Long,
val name:String,
var name:String,
val score: Double
)
app/src/main/java/com/sw/dualscreen/utils/BitmapSaver.kt
+4 -1
View File
@@ -30,7 +30,10 @@ object BitmapSaver {
quality: Int = 100
): File? {
//val dir = context.getExternalFilesDir(null)
val dir = context.cacheDir
val dir = File(context.cacheDir, "crop")
if (dir.exists().not()) {
dir.mkdirs()
}
return saveBitmap(bitmap, File(dir, fileName), format, quality)
}
app/src/main/java/com/sw/dualscreen/viewmodel/UserViewModel.kt
+5 -2
View File
@@ -23,6 +23,7 @@ import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import timber.log.Timber
import kotlin.math.atan
class UserViewModel : BaseViewModel() {
@@ -210,12 +211,14 @@ class UserViewModel : BaseViewModel() {
}
}
fun getFoodInfo(foodName: String) {
fun getFoodInfo(foodName: String, action:(List<FoodInfo>)-> Unit = {}) {
Timber.d("getFoodInfo")
launchWithLoading {
val response = repository.getFoodInfo(foodName = foodName)
if (parseResponse(response)) {
_identifiedFoodInfoList2.value = response.result ?: emptyList()
val list = response.result ?: emptyList()
_identifiedFoodInfoList2.value = list
action(list)
}
}
}
app/src/main/res/drawable/bg_rescan.xml
+11
View File
@@ -0,0 +1,11 @@
<shape xmlns:android="http://schemas.android.com/apk/res/android"
android:shape="rectangle">
<corners
android:bottomLeftRadius="12dp"
android:bottomRightRadius="12dp"
android:topLeftRadius="12dp"
android:topRightRadius="12dp" />
<stroke
android:color="#ffff9900"
android:width="2dp" />
</shape>
app/src/main/res/layout/activity_food_collection.xml
+3 -2
View File
@@ -116,13 +116,14 @@
<androidx.recyclerview.widget.RecyclerView
android:id="@+id/rvFoodCollection"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:layout_height="0dp"
android:layout_weight="1"
android:layout_marginHorizontal="3dp"
android:layout_marginStart="24dp"
android:layout_marginEnd="24dp"
app:layoutManager="androidx.recyclerview.widget.GridLayoutManager"
app:spanCount="3"
tools:itemCount="6"
tools:itemCount="3"
tools:listitem="@layout/list_item_food_collection" />
<LinearLayout
app/src/main/res/layout/activity_main.xml
+29
View File
@@ -78,6 +78,35 @@
android:layout_marginHorizontal="60dp"
android:layout_weight="1" />
<LinearLayout
android:id="@+id/layout_rescan"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginBottom="140dp"
android:gravity="center"
android:orientation="vertical"
android:visibility="gone">
<TextView
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="未识别出菜品，请重试或手动搜索"
android:textColor="@color/white"
android:textSize="24sp" />
<TextView
android:id="@+id/tv_rescan"
android:layout_width="180dp"
android:layout_height="wrap_content"
android:layout_marginTop="20dp"
android:background="@drawable/bg_rescan"
android:gravity="center"
android:padding="10dp"
android:text="重试"
android:textColor="#ffff9900"
android:textSize="32sp" />
</LinearLayout>
<TextView
android:id="@+id/tv_to_search"
android:layout_width="wrap_content"
app/src/main/res/layout/item_food_info.xml
+3 -2
View File
@@ -12,9 +12,10 @@
android:id="@+id/tv_name"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="肉沫干拌面"
android:maxLines="1"
android:ellipsize="end"
android:gravity="center"
android:maxLines="2"
android:text="肉沫干拌面"
android:textColor="#ff031127"
android:textSize="36sp" />