package com.sw.dualscreen.presentation import android.Manifest import android.app.Presentation import android.content.Context import android.content.pm.PackageManager import android.graphics.Bitmap import android.graphics.Outline import android.graphics.Point import android.hardware.Camera import android.hardware.camera2.CameraDevice import android.hardware.camera2.CameraManager import android.os.Build import android.os.Bundle import android.util.DisplayMetrics import android.util.Log import android.view.Display import android.view.View import android.view.ViewGroup import android.view.ViewOutlineProvider import android.view.ViewTreeObserver import android.widget.Toast import androidx.annotation.RequiresApi import androidx.core.content.ContextCompat import androidx.lifecycle.Observer import androidx.lifecycle.lifecycleScope import com.arcsoft.face.ErrorInfo import com.sw.dualscreen.R import com.sw.dualscreen.activity.MainActivity import com.sw.dualscreen.databinding.ItemUserNutritionBinding import com.sw.dualscreen.databinding.PresentationSecondaryScreenBinding import com.sw.dualscreen.ext.dp import com.sw.dualscreen.ext.format2String import com.sw.dualscreen.ext.maskName import com.sw.dualscreen.model.response.DinnerTypeInfo import com.sw.dualscreen.model.response.FoodInfo import com.sw.dualscreen.model.response.UserNutritionData import com.sw.dualscreen.sdk.SensorScaleUtils import com.sw.dualscreen.utils.Debouncer import com.sw.dualscreen.utils.GlideUtils import com.sw.dualscreen.viewmodel.UserViewModel import com.sw.plate.utils.ToastUtils import com.sw.plate.utils.arcface.ConfigUtil import com.sw.plate.utils.arcface.ErrorCodeUtil import com.sw.plate.utils.arcface.FaceRectTransformer import com.sw.plate.utils.arcface.FaceRectView import com.sw.plate.utils.arcface.FaceRectView.DrawInfo import com.sw.plate.utils.arcface.PreviewConfig import com.sw.plate.utils.arcface.camera.CameraListener import com.sw.plate.utils.arcface.camera.DualCameraHelper import com.sw.plate.utils.arcface.face.constants.LivenessType import com.sw.plate.utils.arcface.face.model.FacePreviewInfo import com.sw.plate.utils.arcface.face.model.RecognizeConfiguration import com.sw.plate.utils.arcface.viewmodel.RecognizeViewModel import kotlinx.coroutines.launch import timber.log.Timber /** * 用户屏幕 */ class SecondaryScreenPresentation( val activity: MainActivity, context: Context, display: Display, val viewModel: UserViewModel, private val onDismissListener: () -> Unit = {}, val recognizeViewModel: RecognizeViewModel ) : Presentation(context, display), ViewTreeObserver.OnGlobalLayoutListener { private lateinit var binding: PresentationSecondaryScreenBinding private var cameraDevice: CameraDevice? = null // private var cameraCaptureSession: CameraCaptureSession? = null private lateinit var cameraManager: CameraManager private var cameraIdList: Array = arrayOf() // 当前食物信息 private var currentFood: FoodInfo? = null private var lastFood: FoodInfo? = null private var userNutritionData: UserNutritionData? = null private var dinnerType: DinnerTypeInfo? = null private lateinit var itemUserNutritionBinding: ItemUserNutritionBinding private lateinit var stepChangeCallback: (Int) -> Unit private var lastCalcResultInfo: UserNutritionUtils.CalcResultInfo? = null private var startSensorScale: Boolean = false private var lastWeight = 0.0 private val debouncer = Debouncer(1000) // 当前步骤 var currentStep: Int = 0 override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) binding = PresentationSecondaryScreenBinding.inflate(layoutInflater) setContentView(binding.root) window?.setBackgroundDrawableResource(android.R.color.transparent) initView() setupArcCamera() // binding.includeUserHeat.ivHeat.showText("1070千卡") registerDataChange() } private fun registerDataChange() { activity.lifecycleScope.launch { viewModel.nutritionData.collect { Timber.d("registerDataChange nutritionData = $it") if (it == null) return@collect userNutritionData = it step3() itemUserNutritionBinding.tvUserName.text = it.userName.maskName() itemUserNutritionBinding.tvRecommendHeat.text = "推荐热量:${it.recommendMin}-${it.recommendMax}" } } activity.lifecycleScope.launch { viewModel.dinnerTypeInfo.collect { dinnerType = it } } } private fun initView() { binding.ivImg.outlineProvider = object : ViewOutlineProvider() { override fun getOutline(view: View, outline: Outline) { outline.setRoundRect(0, 0, view.width, view.height, 12f.dp) } } binding.ivImg.clipToOutline = true itemUserNutritionBinding = binding.includeUserNutrition } override fun onDisplayRemoved() { super.onDisplayRemoved() onDismissListener() } fun updateImage(bitmap: Bitmap) { activity.runOnUiThread { binding.ivImg.setImageBitmap(bitmap) } } /** * 启用虹软人脸失败 */ private fun setupArcCamera() { initArcViewModel() initArcView() openRectInfoDraw = true } /** * 用户1 显示相机预览 */ fun step1() { Timber.d("step1") currentStep = 1 stepChangeCallback(currentStep) binding.tvFoodInfo.text = "菜品识别中" binding.ivFaceBg.visibility = View.GONE binding.flPreview.visibility = View.VISIBLE binding.ivImg.visibility = View.VISIBLE binding.flFace.visibility = View.INVISIBLE pauseCamera() binding.tvBottomTip.visibility = View.VISIBLE itemUserNutritionBinding.clNutritionData.visibility = View.GONE stopSensorScale() viewModel.getDinnerType() } /** * 显示人脸识别 */ fun step2(foodInfo: FoodInfo) { Timber.d("step2") currentStep = 2 stepChangeCallback(currentStep) currentFood = foodInfo resumeCamera() binding.tvFoodInfo.text = foodInfo.foodName binding.ivImg.visibility = View.GONE binding.flPreview.visibility = View.GONE binding.flFace.visibility = View.VISIBLE binding.ivFaceBg.visibility = View.VISIBLE binding.tvFaceTip.visibility = View.VISIBLE binding.tvBottomTip.visibility = View.VISIBLE itemUserNutritionBinding.clNutritionData.visibility = View.GONE } fun updateFood(foodInfo: FoodInfo?) { Timber.d("updateFood") currentFood = foodInfo if (foodInfo != null) { if (currentStep == 1) { step2(foodInfo) } else { binding.tvFoodInfo.text = foodInfo.foodName GlideUtils.loadRoundCornerImage( context, url = currentFood!!.imgUrl, imageView = binding.ivImg, radius = 12 ) } } } /** * 显示识别出的菜品信息 */ fun step3() { Timber.d("step3") currentStep = 3 stepChangeCallback(currentStep) if (currentFood == null) { ToastUtils.showToast("选中物品为空3") return } binding.tvFoodInfo.text = currentFood!!.foodName pauseCamera() startSensorScale() GlideUtils.loadRoundCornerImage( context, url = currentFood!!.imgUrl, imageView = binding.ivImg, radius = 12 ) binding.flPreview.visibility = View.VISIBLE binding.ivImg.visibility = View.VISIBLE binding.flFace.visibility = View.INVISIBLE binding.ivFaceBg.visibility = View.GONE binding.tvFaceTip.visibility = View.GONE binding.tvBottomTip.visibility = View.GONE itemUserNutritionBinding.clNutritionData.visibility = View.VISIBLE } private fun startSensorScale() { Timber.d("startSensorScale startSensorScale = $startSensorScale") if (startSensorScale) return SensorScaleUtils.startScale { weight -> if (currentFood == null) { Timber.e("startSensorScale currentFood is null") return@startScale } if (userNutritionData == null) { Timber.e("startSensorScale userNutritionData is null") return@startScale } if (dinnerType == null) { Timber.e("startSensorScale dinnerType is null") return@startScale } // 重量和 当前食物都没变,则不再计算 if (weight == lastWeight && lastFood == currentFood) return@startScale Timber.d("registerDataChange weight = $weight") lastFood = currentFood lastWeight = weight debouncer.debounce { val calcResultInfo = UserNutritionUtils.calculateNutrition( currentFood!!, userNutritionData!!, weight, dinnerType = dinnerType!!.dinnerType!!.dinnerType!! ) Timber.d("calcResultInfo = $calcResultInfo") if (lastCalcResultInfo == calcResultInfo) { return@debounce } lastCalcResultInfo = calcResultInfo activity.runOnUiThread { val totalKcal = calcResultInfo.totalKcal val grain = calcResultInfo.grain val fruits = calcResultInfo.fruits val meat = calcResultInfo.meat // 设置热量 itemUserNutritionBinding.customKcalColumn.setImageDrawable( true, totalKcal.first < totalKcal.second ) itemUserNutritionBinding.totalKcalTv.text = totalKcal.second.format2String() itemUserNutritionBinding.customKcalColumn.setCustomHeightPercent( (totalKcal.second / 100).toFloat(), true ) // 设置主食 itemUserNutritionBinding.customFoodColumn.setImageDrawable( false, grain.first < grain.second ) itemUserNutritionBinding.totalFoodTv.text = grain.second.format2String() itemUserNutritionBinding.customFoodColumn.setCustomHeightPercent( (grain.second / 100).toFloat(), true ) // 设置果蔬 itemUserNutritionBinding.customVegetableColumn.setImageDrawable( false, fruits.first < fruits.second ) itemUserNutritionBinding.totalVegetableTv.text = fruits.second.format2String() itemUserNutritionBinding.customVegetableColumn.setCustomHeightPercent( (fruits.second / 100).toFloat(), true ) // 设置肉蛋 itemUserNutritionBinding.customMeatColumn.setImageDrawable( false, meat.first < meat.second ) itemUserNutritionBinding.totalMeatTv.text = meat.second.format2String() itemUserNutritionBinding.customMeatColumn.setCustomHeightPercent( (meat.second / 100).toFloat(), true ) } } } } private fun stopSensorScale() { SensorScaleUtils.closeScale() } private fun openCamera() { Timber.d("openCamera") if (ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED ) { ToastUtils.showToast("无摄像头权限") return } try { cameraManager = context.getSystemService(Context.CAMERA_SERVICE) as CameraManager // 获取摄像头列表 cameraIdList = cameraManager.cameraIdList if (cameraIdList.isEmpty()) { Toast.makeText(context, "没有可用的摄像头", Toast.LENGTH_SHORT).show() return } cameraIdList.forEach { Timber.d("摄像头id列表 = $it") } val cameraId = cameraIdList.last() Timber.d("openCamera cameraId = $cameraId") cameraManager.openCamera(cameraId, object : CameraDevice.StateCallback() { override fun onOpened(device: CameraDevice) { cameraDevice = device // createCameraPreviewSession() recognizeViewModel.init( PreviewConfig( 2, 4, 0, 0 ) ) initRgbCamera() resumeCamera() } override fun onDisconnected(device: CameraDevice) { device.close() cameraDevice = null } override fun onError(device: CameraDevice, error: Int) { device.close() cameraDevice = null } }, null) } catch (e: Exception) { Log.e("Camera2", "打开摄像头失败", e) } } fun setStepChangeCallback(callback: (Int) -> Unit) { stepChangeCallback = callback } // 虹软人脸配置 ⬇ private var isRecognition = false private var rgbCameraHelper: DualCameraHelper? = null private var rgbFaceRectTransformer: FaceRectTransformer? = null private val livenessType = LivenessType.RGB private var openRectInfoDraw = false private fun initArcViewModel() { recognizeViewModel.setLiveType(livenessType) recognizeViewModel.ftInitCode.observe(activity, Observer { ftInitCode: Int? -> if (ftInitCode != ErrorInfo.MOK) { val error: String? = context.getString( R.string.specific_engine_init_failed, "ftEngine", ftInitCode, ErrorCodeUtil.arcFaceErrorCodeToFieldName(ftInitCode!!) ) Timber.e("ftInitCode observe = $error") ToastUtils.showToast(error) } }) recognizeViewModel.frInitCode.observe(activity, Observer { frInitCode: Int? -> if (frInitCode != ErrorInfo.MOK) { val error: String? = context.getString( R.string.specific_engine_init_failed, "frEngine", frInitCode, ErrorCodeUtil.arcFaceErrorCodeToFieldName(frInitCode!!) ) Timber.e("frInitCode observe = $error") ToastUtils.showToast(error) } }) recognizeViewModel.flInitCode.observe(activity, Observer { flInitCode: Int? -> if (flInitCode != ErrorInfo.MOK) { val error: String? = context.getString( R.string.specific_engine_init_failed, "flEngine", flInitCode, ErrorCodeUtil.arcFaceErrorCodeToFieldName(flInitCode!!) ) Timber.e("flInitCode observe = $error") ToastUtils.showToast(error) } }) recognizeViewModel.recognizeConfiguration .observe(activity, Observer { recognizeConfiguration: RecognizeConfiguration? -> Timber.i("recognizeConfiguration: observe = ${recognizeConfiguration.toString()}") }) recognizeViewModel.recognizeNotice.observe(activity, Observer { notice: String? -> Timber.i("recognizeNotice observe notice = $notice") if (currentStep == 3 && notice == "leave") { // step1() } }) recognizeViewModel.recognizeUserId.observe(activity, Observer { userId: String? -> Timber.i("recognizeUserId observe userId = $userId") if (userId == null) return@Observer viewModel.getUserNutritionData(userId = userId, foodId = currentFood!!.id!!) }) recognizeViewModel.drawRectInfoText.observe(activity, Observer { info -> Timber.i("drawRectInfoText observe info = $info") }) } private fun initArcView() { //在布局结束后才做初始化操作 binding.dualCameraTexturePreviewRgb.getViewTreeObserver().addOnGlobalLayoutListener(this) recognizeViewModel.getCompareResultList().getValue() } /** * 调整View的宽高,使预览显示正常且采集框固定为380x380 * * @param rgbPreview RGB预览View * @param previewView 显示预览数据的view * @param faceRectView 画框的view (380x380) * @param previewSize 预览大小 * @param displayOrientation 相机旋转角度 * @param scale 缩放比例 * @return 调整后的LayoutParams */ private fun adjustPreviewViewSize( rgbPreview: View, previewView: View, faceRectView: FaceRectView, previewSize: Camera.Size, displayOrientation: Int, scale: Float ): ViewGroup.LayoutParams { val layoutParams = previewView.layoutParams val measuredWidth = previewView.measuredWidth val measuredHeight = previewView.measuredHeight var ratio = (previewSize.height.toFloat()) / previewSize.width.toFloat() if (ratio > 1) { ratio = 1 / ratio } if (displayOrientation % 180 == 0) { layoutParams.width = measuredWidth layoutParams.height = (measuredWidth * ratio).toInt() } else { layoutParams.height = measuredHeight layoutParams.width = (measuredHeight * ratio).toInt() } if (scale < 1f) { val rgbParam = rgbPreview.getLayoutParams() layoutParams.width = (rgbParam.width * scale).toInt() layoutParams.height = (rgbParam.height * scale).toInt() } else { layoutParams.width = (layoutParams.width * scale).toInt() layoutParams.height = (layoutParams.height * scale).toInt() } val metrics = DisplayMetrics() activity.windowManager.defaultDisplay.getMetrics(metrics) if (layoutParams.width >= metrics.widthPixels) { val viewRatio = layoutParams.width / (metrics.widthPixels.toFloat()) layoutParams.width = (layoutParams.width / viewRatio).toInt() layoutParams.height = (layoutParams.height / viewRatio).toInt() } if (layoutParams.height >= metrics.heightPixels) { val viewRatio = layoutParams.height / (metrics.heightPixels.toFloat()) layoutParams.width = (layoutParams.width / viewRatio).toInt() layoutParams.height = (layoutParams.height / viewRatio).toInt() } previewView.setLayoutParams(layoutParams) faceRectView.setLayoutParams(layoutParams) return layoutParams } private fun initRgbCamera() { val cameraListener: CameraListener = object : CameraListener { override fun onCameraOpened( camera: Camera, cameraId: Int, displayOrientation: Int, isMirror: Boolean ) { activity.runOnUiThread({ val previewSizeRgb = camera.getParameters().getPreviewSize() val layoutParams = adjustPreviewViewSize( binding.dualCameraTexturePreviewRgb, binding.dualCameraTexturePreviewRgb, binding.dualCameraFaceRectView, previewSizeRgb, 90, 1f ) Timber.d("initRgbCamera previewSizeRgb = ${previewSizeRgb.width}-${previewSizeRgb.height}") Timber.d("initRgbCamera layoutParams = ${layoutParams.width}-${layoutParams.height}") rgbFaceRectTransformer = FaceRectTransformer( previewSizeRgb.width, previewSizeRgb.height, layoutParams.width, layoutParams.height, displayOrientation, cameraId, isMirror, ConfigUtil.isDrawRgbRectHorizontalMirror(context), ConfigUtil.isDrawRgbRectVerticalMirror(context) ) recognizeViewModel.onRgbCameraOpened(camera) recognizeViewModel.setRgbFaceRectTransformer(rgbFaceRectTransformer) }) } @RequiresApi(api = Build.VERSION_CODES.Q) override fun onPreview(nv21: ByteArray?, camera: Camera?) { if (!isRecognition) { return } binding.dualCameraFaceRectView.clearFaceInfo() val facePreviewInfoList: MutableList? = recognizeViewModel.onPreviewFrame(nv21, true) if (facePreviewInfoList != null && rgbFaceRectTransformer != null) { drawPreviewInfo(facePreviewInfoList) } recognizeViewModel.clearLeftFace(facePreviewInfoList) } override fun onCameraClosed() { Timber.i("onCameraClosed: ") } override fun onCameraError(e: java.lang.Exception) { Timber.i("onCameraError: %s", e.message) e.printStackTrace() } override fun onCameraConfigurationChanged(cameraID: Int, displayOrientation: Int) { Timber.i("onCameraConfigurationChanged: threadName = ${Thread.currentThread().name}") if (rgbFaceRectTransformer != null) { rgbFaceRectTransformer!!.cameraDisplayOrientation = displayOrientation } Timber.i("onCameraConfigurationChanged: $cameraID $displayOrientation") } } val previewConfig: PreviewConfig = recognizeViewModel.previewConfig rgbCameraHelper = DualCameraHelper.Builder() .previewViewSize( Point( binding.dualCameraTexturePreviewRgb.measuredWidth, binding.dualCameraTexturePreviewRgb.measuredHeight ) ) .rotation(activity.windowManager.defaultDisplay.rotation) .additionalRotation(90) // 角度 .previewSize(recognizeViewModel.loadPreviewSize()) .specificCameraId(previewConfig.rgbCameraId) .isMirror(true) .previewOn(binding.dualCameraTexturePreviewRgb) .cameraListener(cameraListener) .build() rgbCameraHelper!!.init() rgbCameraHelper!!.start() } /** * 绘制RGB、IR画面的实时人脸信息 * * @param facePreviewInfoList RGB画面的实时人脸信息 */ private fun drawPreviewInfo(facePreviewInfoList: MutableList) { if (rgbFaceRectTransformer != null) { val rgbDrawInfoList: MutableList? = recognizeViewModel.getDrawInfo( facePreviewInfoList, LivenessType.RGB, openRectInfoDraw ) // 识别成功 binding.dualCameraFaceRectView.drawRealtimeFaceInfo(rgbDrawInfoList) } } fun openRectInfoDraw(view: View?) { openRectInfoDraw = !openRectInfoDraw recognizeViewModel.setDrawRectInfoTextValue(openRectInfoDraw) } override fun onGlobalLayout() { Timber.d("onGlobalLayout") binding.dualCameraTexturePreviewRgb.getViewTreeObserver().removeOnGlobalLayoutListener(this) openCamera() } // override fun onResume() { // super.onResume() // resumeCamera() // viewModel.resetUserInfo() // } // // protected override fun onPause() { // pauseCamera() // super.onPause() // } override fun onStop() { if (rgbCameraHelper != null) { rgbCameraHelper!!.release() rgbCameraHelper = null } SensorScaleUtils.closeScale() recognizeViewModel.destroy() super.onStop() } fun resumeCamera() { Timber.d("resumeCamera isRecognition = $isRecognition") isRecognition = true if (rgbCameraHelper != null && rgbCameraHelper!!.isStopped) { rgbCameraHelper!!.start() } } fun pauseCamera() { Timber.d("pauseCamera isRecognition = $isRecognition") isRecognition = false recognizeViewModel.onPreviewFrame(ByteArray(1382400), true) } }