Merge branch 'dev_lvmeng_1016'

# Conflicts:
#	app/build.gradle.kts
#	build.gradle.kts
#	settings.gradle.kts
This commit is contained in:
2025-10-16 19:30:22 +08:00
16 changed files with 477 additions and 5 deletions
+1 -1
View File
@@ -6,7 +6,7 @@
<GradleProjectSettings>
<option name="testRunner" value="CHOOSE_PER_TEST" />
<option name="externalProjectPath" value="$PROJECT_DIR$" />
<option name="gradleJvm" value="#GRADLE_LOCAL_JAVA_HOME" />
<option name="gradleJvm" value="zulu-17" />
<option name="modules">
<set>
<option value="$PROJECT_DIR$" />
Binary file not shown.
Binary file not shown.

After

Width:  |  Height:  |  Size: 515 KiB

+41
View File
@@ -0,0 +1,41 @@
{
"class_names": [
"回锅肉",
"手撕包菜",
"炒细面",
"红烧肉",
"肉沫酸豆角",
"蒜苔炒肉",
"西红柿鸡蛋",
"豆角茄子",
"麻婆豆腐",
"麻辣小面"
],
"class_to_idx": {
"回锅肉": 0,
"手撕包菜": 1,
"炒细面": 2,
"红烧肉": 3,
"肉沫酸豆角": 4,
"蒜苔炒肉": 5,
"西红柿鸡蛋": 6,
"豆角茄子": 7,
"麻婆豆腐": 8,
"麻辣小面": 9
},
"idx_to_class": {
"0": "回锅肉",
"1": "手撕包菜",
"2": "炒细面",
"3": "红烧肉",
"4": "肉沫酸豆角",
"5": "蒜苔炒肉",
"6": "西红柿鸡蛋",
"7": "豆角茄子",
"8": "麻婆豆腐",
"9": "麻辣小面"
},
"embedding_dim": 512,
"total_images": 500,
"build_time": "2025-10-16T17:51:06.285784"
}
File diff suppressed because one or more lines are too long
+1
View File
@@ -0,0 +1 @@
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]
+2 -1
View File
@@ -1,5 +1,6 @@
package com.sw.dualscreen
import com.sw.dualscreen.objbox.ObjectBox
import com.sw.dualscreen.utils.CrashHandler
import com.sw.plate.App
import com.sw.plate.utils.AppUtil
@@ -17,7 +18,7 @@ class MyApp : App() {
Timber.d("UDID = ${AppUtil.getUDID(this)}")
deviceId = "3ea47dc0-3cf0-3c2f-909c-265a9a65572e"
GlobalData.deviceId = deviceId
ObjectBox.init(this)
// 初始化崩溃处理器
CrashHandler.init(this)
}
@@ -10,6 +10,7 @@ import android.view.View
import android.view.ViewOutlineProvider
import android.widget.AdapterView
import android.widget.ArrayAdapter
import android.widget.Toast
import androidx.activity.viewModels
import androidx.camera.core.CameraSelector
import androidx.camera.core.ImageAnalysis
@@ -29,10 +30,13 @@ import com.sw.dualscreen.databinding.ActivityMainBinding
import com.sw.dualscreen.databinding.ItemFoodInfoBinding
import com.sw.dualscreen.ext.dp
import com.sw.dualscreen.model.response.FoodInfo
import com.sw.dualscreen.objbox.FoodModule
import com.sw.dualscreen.presentation.SecondaryScreenPresentation
import com.sw.dualscreen.sdk.SensorScaleUtils
import com.sw.dualscreen.utils.AssetsTool
import com.sw.dualscreen.utils.Debouncer
import com.sw.dualscreen.utils.GlideUtils
import com.sw.dualscreen.utils.ImageUtil
import com.sw.dualscreen.view.CustomBottomSheetDialog
import com.sw.dualscreen.viewmodel.BaseViewModel
import com.sw.dualscreen.viewmodel.UserViewModel
@@ -76,6 +80,7 @@ class MainActivity : BaseActivity<ActivityMainBinding>() {
setupSecondaryDisplay()
setupCamera()
initData()
FoodModule.init(this)
}
private fun initData() {
@@ -184,7 +189,23 @@ class MainActivity : BaseActivity<ActivityMainBinding>() {
debouncer.debounce {
takePhoto { photoUri ->
lastPhotoUri = photoUri
viewModel.getIdentifiedFoodList()
Timber.d("registerDataChange photoUri = ${photoUri.path}")
// viewModel.getIdentifiedFoodList()
AssetsTool.loadImageBitmapFromAssets(this, "data/20251016-091510.png") { bitmap ->
val resultList = FoodModule.queryFood(bitmap)
val name = resultList.joinToString(separator = ",") { it.name.toString() }
Timber.d("registerDataChange 固定图片识别数据名称:$name")
}
ImageUtil.uriToBitmap(this, photoUri)?.let { bitmap ->
Timber.d("registerDataChange photoUri 拿到bitmap")
val resultList = FoodModule.queryFood(bitmap)
Timber.d("registerDataChange photoUri 拿到识别数据")
val name = resultList.joinToString(separator = ",") { it.name.toString() }
Timber.d("registerDataChange photoUri 识别数据名称:$name")
// Toast.makeText(this, "${res.name}|${res.foodIdx}|${res.score}", Toast.LENGTH_LONG).show()
}
}
}
}
@@ -0,0 +1,14 @@
package com.sw.dualscreen.objbox
data class FoodClassInfo(
var class_names: List<String>,
var class_to_idx: Map<String, Int>,
var idx_to_class: Map<String, String>
)
data class FoodQueryResult(
var id: Long,
var name: String?,
var foodIdx: Int,
var score: Double
)
@@ -0,0 +1,37 @@
package com.sw.dualscreen.objbox
import io.objectbox.annotation.Entity
import io.objectbox.annotation.HnswIndex
import io.objectbox.annotation.Id
import io.objectbox.annotation.VectorDistanceType
@Entity
data class Food(
@Id var id: Long = 0,
var name: String? = null,
var foodIdx: Int = 0,
@HnswIndex(dimensions = 512, distanceType = VectorDistanceType.DOT_PRODUCT)
var foodVector: FloatArray? = null
) {
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as Food
if (id != other.id) return false
if (foodIdx != other.foodIdx) return false
if (name != other.name) return false
if (!foodVector.contentEquals(other.foodVector)) return false
return true
}
override fun hashCode(): Int {
var result = id.hashCode()
result = 31 * result + foodIdx
result = 31 * result + (name?.hashCode() ?: 0)
result = 31 * result + (foodVector?.contentHashCode() ?: 0)
return result
}
}
@@ -0,0 +1,146 @@
package com.sw.dualscreen.objbox
import android.content.Context
import android.graphics.Bitmap
import com.google.gson.Gson
import com.google.gson.reflect.TypeToken
import com.sw.dualscreen.utils.AssetsTool
import io.objectbox.Box
import io.objectbox.kotlin.boxFor
import io.objectbox.query.Query
import org.pytorch.IValue
import org.pytorch.Module
import org.pytorch.torchvision.TensorImageUtils
import java.io.File
import java.io.FileOutputStream
import java.io.IOException
import java.io.InputStream
object FoodModule {
private lateinit var module_mobile: Module
private lateinit var box: Box<Food>
private lateinit var embeddingsList: List<List<Float>>
private lateinit var labelsList: IntArray
private lateinit var classInfo: FoodClassInfo
private val NO_MEAN_RGB = floatArrayOf(0.0f, 0.0f, 0.0f)
private val NO_STD_RGB = floatArrayOf(1.0f, 1.0f, 1.0f)
fun init(context: Context) {
module_mobile = Module.load(copyAssetToCache(context, "best_embedding_model_mobile.pt"))
box = ObjectBox.boxStore.boxFor(Food::class)
if (box.all.isNotEmpty()) {
box.removeAll()
}
if (box.all.isEmpty()) {
initFoodData(context)
}
}
fun queryFood(bitmap: Bitmap, queryCount:Int = 30): MutableList<FoodQueryResult> {
val inputTensor = TensorImageUtils.bitmapToFloat32Tensor(
bitmap,
NO_MEAN_RGB, // [0.485, 0.456, 0.406] TORCHVISION_NORM_MEAN_RGB
NO_STD_RGB // [0.229, 0.224, 0.225] TORCHVISION_NORM_STD_RGB
)
val outputTensor = module_mobile.forward(IValue.from(inputTensor)).toTensor()
val floatArray = outputTensor.dataAsFloatArray
return queryFood(floatArray,queryCount)
}
fun queryFood(floatArray: FloatArray, queryCount:Int = 30): MutableList<FoodQueryResult> {
val query: Query<Food> = box.query(Food_.foodVector.nearestNeighbors(floatArray, queryCount)).build()
//查询比较分数
// val tempList = query.findWithScores().sortedBy { it.score }.map { "${it.get().name}|${it.get().foodIdx}|${it.score}" }
val resultList = mutableListOf<FoodQueryResult>()
query.findIdsWithScores().forEach {
val food = box.get(it.id)
resultList.add(
FoodQueryResult(
id = it.id,
name = food.name,
foodIdx = food.foodIdx,
score = it.score
)
)
}
return resultList
}
private fun initFoodData(context: Context) {
if (box.all.isNotEmpty()) {
return
}
val embeddingsJson = AssetsTool.readAssetsFile(context, "data/embeddings.json")
val labelsJson = AssetsTool.readAssetsFile(context, "data/labels.json")
val classInfoJson = AssetsTool.readAssetsFile(context, "data/class_info.json")
embeddingsList =
Gson().fromJson(embeddingsJson, object : TypeToken<List<List<Float>>>() {}.type)
labelsList = Gson().fromJson(labelsJson, IntArray::class.java)
classInfo =
Gson().fromJson(classInfoJson, FoodClassInfo::class.java)
val foodMap = classInfo.idx_to_class
embeddingsList.forEachIndexed { index, floatList ->
val classIdx = labelsList[index]
val foodName = foodMap["$classIdx"]
val array = floatList.toFloatArray()
box.put(Food(name = foodName, foodVector = array, foodIdx = index))
}
}
/**
* ,此方法的主要目的是:从assets 拷贝到 app的cache目录
* @param context
* @param fileName
* @return 例如是这样:/data/user/0/com.frizzle.pluginhookandroid9/cache/plugin-debug.apk
*
* 不可能反正SD
*/
fun copyAssetToCache(context: Context, fileName: String): String? {
// 此app的缓存目录 --> 会默认在 cache目录...,可以自己去看看哦
val cacheDir = context.getCacheDir()
if (!cacheDir.exists()) {
cacheDir.mkdirs() // TODO 如果没有缓存目录,就创建
}
val outPath = File(cacheDir, fileName) // TODO 创建输出的文件位置
if (outPath.exists()) {
outPath.delete() // TODO 如果该文件已经存在,就删掉
}
var `is`: InputStream? = null // 读取
var fos: FileOutputStream? = null // 写入
try {
// 创建文件,如果创建成功,就返回true
val res = outPath.createNewFile()
if (res) {
`is` = context.getAssets().open(fileName) // 拿到main/assets目录的输入流,用于读取字节
fos = FileOutputStream(outPath) // 读取出来的字节最终写到outPath
val buf = ByteArray(`is`.available()) // 缓存区
var byteCount: Int
// 开始循环读取
while ((`is`.read(buf).also { byteCount = it }) != -1) {
fos.write(buf, 0, byteCount)
}
return outPath.getAbsolutePath()
}
} catch (e: IOException) {
e.printStackTrace()
} finally {
try {
// TODO 一定要记得关闭资源,为了不去性能的磨损
fos!!.flush()
`is`!!.close()
fos.close()
} catch (e: IOException) {
e.printStackTrace()
}
}
return null
}
}
@@ -0,0 +1,107 @@
/*
* Copyright 2024 ObjectBox Ltd. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.sw.dualscreen.objbox
import android.content.Context
import android.util.Log
import io.objectbox.BoxStore
import io.objectbox.BoxStoreBuilder
import io.objectbox.android.Admin
import io.objectbox.android.ObjectBoxLiveData
import io.objectbox.exception.DbException
import io.objectbox.exception.FileCorruptException
import io.objectbox.sync.Sync
import java.io.File
import java.util.Date
import java.util.zip.GZIPOutputStream
import kotlin.also
import kotlin.io.copyTo
import kotlin.io.inputStream
import kotlin.io.outputStream
import kotlin.io.use
import kotlin.jvm.java
import kotlin.jvm.javaClass
/**
* Singleton to keep BoxStore reference and provide current list of Notes Objects.
* Inserts demo data if no Objects are stored.
*/
object ObjectBox {
private const val TAG = "ObjectBox"
lateinit var boxStore: BoxStore
private set
/**
* If building the [boxStore] failed, contains the thrown error message.
*/
var dbExceptionMessage: String? = null
private set
fun init(context: Context) {
// On Android make sure to pass a Context when building the Store.
boxStore = try {
MyObjectBox.builder()
.androidContext(context.applicationContext)
.build()
} catch (e: DbException) {
if (e.javaClass.equals(DbException::class.java) || e is FileCorruptException) {
// Failed to build BoxStore due to database file issue, store message;
// checked in NoteListActivity to notify user.
dbExceptionMessage = e.toString()
return
} else {
// Failed to build BoxStore due to developer error.
throw e
}
}
// if (BuildConfig.DEBUG) {
var syncAvailable = if (Sync.isAvailable()) "available" else "unavailable"
Log.d(TAG,"Using ObjectBox ${BoxStore.getVersion()} (${BoxStore.getVersionNative()}, sync $syncAvailable)")
// Enable ObjectBox Admin on debug builds.
// https://docs.objectbox.io/data-browser
Admin(boxStore).start(context.applicationContext)
// }
}
/**
* If the database file is not in use, compresses (GZIP) and copies it to the given [target].
*/
fun copyAndGzipDatabaseFileTo(target: File, context: Context): Boolean {
if (BoxStore.isDatabaseOpen(context, null)) {
// Do not copy if database file is still in use.
// If it would be open, the copy will likely get corrupted
// as BoxStore may currently write data to the file.
Log.e(TAG, "Database file is still in use, can not copy.")
return false
}
// If a name was given when building BoxStore use that instead of the default below.
val dbName = BoxStoreBuilder.DEFAULT_NAME
File(context.filesDir, "objectbox/$dbName/data.mdb").inputStream().use { input ->
target.parentFile?.mkdirs()
GZIPOutputStream(target.outputStream()).use { output ->
input.copyTo(output, DEFAULT_BUFFER_SIZE)
}
}
return true
}
}
@@ -0,0 +1,76 @@
package com.sw.dualscreen.utils
import android.content.Context
import android.graphics.Bitmap
import android.graphics.BitmapFactory
import java.io.BufferedReader
import java.io.IOException
import java.io.InputStreamReader
object AssetsTool {
// fun readJson(context: Context, fileName: String): List<String> {
// val json = readAssetsFile(context, fileName)
// val json2 = json.replace("[[", "[").replaceAfterLast("]]", "]")
// val list = json2.split("],")
// val resultList = mutableListOf<String>()
// var count = 0
// val tempList = mutableListOf<String>()
// list.forEachIndexed { index, text ->
// if (count >= 500) {
// val tempJson = "[${tempList.joinToString (",")}]"
// resultList.add(tempJson)
// tempList.clear()
// count = 0
// }
// val newText = if (index == list.size - 1) text else "${text}]"
// tempList.add(newText)
// count++
// }
// if (count < 500) {
// val tempJson = "[${tempList.joinToString(",")}]"
// resultList.add(tempJson)
// }
// return resultList
// }
fun readAssetsFile(context: Context, fileName: String): String {
val stringBuilder = StringBuilder()
try {
val bf = BufferedReader(InputStreamReader(context.assets.open(fileName)))
bf.useLines { lines -> lines.forEach { stringBuilder.append(it) } }
} catch (e: IOException) {
e.printStackTrace()
}
return stringBuilder.toString()
}
fun loadImagesFromAssets(context: Context, subPath: String): MutableList<Bitmap> {
val bitmaps: MutableList<Bitmap> = mutableListOf()
val assetManager = context.assets
try {
val files = assetManager.list(subPath)
files?.forEach { file ->
assetManager.open("$subPath/$file").use { `is` ->
val bitmap = BitmapFactory.decodeStream(`is`)
if (bitmap != null) {
bitmaps.add(bitmap)
}
}
}
} catch (e: IOException) {
e.printStackTrace()
}
return bitmaps
}
fun loadImageBitmapFromAssets(context: Context, imagePath:String, action:(bmp: Bitmap)-> Unit) {
context.assets.open(imagePath).use { `is` ->
val bitmap = BitmapFactory.decodeStream(`is`)
action(bitmap)
}
}
}
@@ -0,0 +1,21 @@
package com.sw.dualscreen.utils
import android.content.Context
import android.graphics.Bitmap
import android.graphics.BitmapFactory
import android.net.Uri
object ImageUtil {
fun uriToBitmap(context: Context, uri: Uri): Bitmap? {
return try {
context.contentResolver.openInputStream(uri)?.use { stream ->
BitmapFactory.decodeStream(stream)
}
} catch (e: Exception) {
e.printStackTrace()
null
}
}
}
+2 -1
View File
@@ -2,6 +2,7 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
#distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip
distributionUrl=https\://mirrors.cloud.tencent.com/gradle/gradle-8.11.1-all.zip
#distributionUrl=https\://mirrors.cloud.tencent.com/gradle/gradle-8.11.1-all.zip
distributionUrl=https\://mirrors.cloud.tencent.com/gradle/gradle-8.14.3-all.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
+6 -1
View File
@@ -1,5 +1,6 @@
plugins {
alias(libs.plugins.android.library)
// alias(libs.plugins.android.library)
id("com.android.library")
}
android {
@@ -64,4 +65,8 @@ dependencies {
val glideVersion = "4.12.0"
implementation("com.github.bumptech.glide:glide:$glideVersion")
annotationProcessor("com.github.bumptech.glide:compiler:$glideVersion")
val objectboxVersion = "5.0.1"
debugImplementation("io.objectbox:objectbox-android-objectbrowser:$objectboxVersion")
// releaseImplementation("io.objectbox:objectbox-android:$objectboxVersion")
}