Skip to content

Commit 5ffa30d

Browse files
rohit9625rohit9625
authored
feat: implement capture, analyze and translate text from photo (#9)
* feat: implement capture photo function * feat: navigate to view image screen after successful image capture * feat: show translation state in the bottom sheet at view image scren * feat: add function to analyze a single bitmap image * refactor: expose detected text lines via state flow from TextAnalyzer * fix: close text analyzer and detach camera controller correctly * refactor: localize string resources
1 parent acd789f commit 5ffa30d

11 files changed

Lines changed: 347 additions & 60 deletions

File tree

‎app/src/main/java/dev/androhit/natively/camera/data/CameraController.kt‎

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,18 @@
11
package dev.androhit.natively.camera.data
22

33
import android.content.Context
4-
import android.net.Uri
4+
import android.graphics.Bitmap
5+
import android.graphics.Matrix
56
import androidx.camera.core.CameraSelector
67
import androidx.camera.core.ImageCapture
78
import androidx.camera.core.ImageCaptureException
9+
import androidx.camera.core.ImageProxy
810
import androidx.camera.mlkit.vision.MlKitAnalyzer
911
import androidx.camera.view.CameraController.IMAGE_ANALYSIS
1012
import androidx.camera.view.CameraController.IMAGE_CAPTURE
1113
import androidx.camera.view.LifecycleCameraController
1214
import androidx.core.content.ContextCompat
1315
import androidx.lifecycle.LifecycleOwner
14-
import java.io.File
1516

1617
class CameraController(
1718
private val context: Context
@@ -48,17 +49,26 @@ class CameraController(
4849
}
4950

5051
fun capturePhoto(
51-
outputFile: File,
52-
onResult: (Result<Uri>) -> Unit
52+
onResult: (Result<Bitmap>) -> Unit
5353
) {
54-
val options = ImageCapture.OutputFileOptions.Builder(outputFile).build()
55-
5654
controller.takePicture(
57-
options,
5855
ContextCompat.getMainExecutor(context),
59-
object : ImageCapture.OnImageSavedCallback {
60-
override fun onImageSaved(result: ImageCapture.OutputFileResults) {
61-
onResult(Result.success(result.savedUri!!))
56+
object : ImageCapture.OnImageCapturedCallback() {
57+
override fun onCaptureSuccess(image: ImageProxy) {
58+
super.onCaptureSuccess(image)
59+
val matrix = Matrix().apply {
60+
postRotate(image.imageInfo.rotationDegrees.toFloat())
61+
}
62+
val rotatedBitmap = Bitmap.createBitmap(
63+
image.toBitmap(),
64+
0,
65+
0,
66+
image.width,
67+
image.height,
68+
matrix,
69+
true
70+
)
71+
onResult(Result.success(rotatedBitmap))
6272
}
6373

6474
override fun onError(exception: ImageCaptureException) {

‎app/src/main/java/dev/androhit/natively/camera/ui/CameraScreen.kt‎

Lines changed: 6 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@ import androidx.compose.ui.graphics.Color
3939
import androidx.compose.ui.graphics.drawscope.Stroke
4040
import androidx.compose.ui.input.pointer.pointerInput
4141
import androidx.compose.ui.layout.onGloballyPositioned
42-
import androidx.compose.ui.platform.LocalContext
4342
import androidx.compose.ui.res.painterResource
4443
import androidx.compose.ui.unit.IntOffset
4544
import androidx.compose.ui.unit.IntSize
@@ -48,7 +47,6 @@ import androidx.lifecycle.compose.collectAsStateWithLifecycle
4847
import dev.androhit.natively.R
4948
import dev.androhit.natively.camera.data.CameraController
5049
import dev.androhit.natively.camera.ui.components.CameraPreview
51-
import dev.androhit.natively.data.TextAnalyzer
5250
import dev.androhit.natively.domain.RecognizedText
5351
import dev.androhit.natively.domain.TextScript
5452
import dev.androhit.natively.ui.components.CameraFeature
@@ -57,35 +55,28 @@ import dev.androhit.natively.ui.components.SwitchFeatureBottomBar
5755
import dev.androhit.natively.ui.components.TranslateTextChip
5856
import dev.androhit.natively.ui.states.Language
5957
import dev.androhit.natively.ui.states.TranslationState
60-
import org.koin.compose.viewmodel.koinViewModel
6158

6259
@Composable
6360
fun CameraScreen(
6461
cameraController: CameraController,
62+
viewModel: CameraViewModel,
63+
onViewImage: () -> Unit = {},
6564
script: TextScript? = null,
6665
) {
67-
val context = LocalContext.current.applicationContext
68-
val viewModel = koinViewModel<CameraViewModel>()
69-
7066
val selectedFeature by viewModel.selectedFeature.collectAsStateWithLifecycle()
7167
val detectedTextLines by viewModel.detectedTextLines.collectAsStateWithLifecycle()
7268
val translationState by viewModel.translationState.collectAsStateWithLifecycle()
73-
74-
val textAnalyzer = remember {
75-
TextAnalyzer(context, viewModel::onTextDetected)
76-
}
7769

7870
LaunchedEffect(selectedFeature, script) {
7971
if (selectedFeature == CameraFeature.LiveTranslate) {
80-
script?.let { textAnalyzer.updateRecognizer(script) }
81-
viewModel.attachTextAnalyzer(textAnalyzer.getInstance())
72+
script?.let { viewModel.updateScript(script) }
73+
viewModel.attachTextAnalyzer()
8274
}
8375
}
8476

8577
DisposableEffect(Unit) {
8678
onDispose {
87-
cameraController.detach()
88-
textAnalyzer.close()
79+
viewModel.cleanUp()
8980
}
9081
}
9182

@@ -161,7 +152,7 @@ fun CameraScreen(
161152
exit = fadeOut() + scaleOut()
162153
) {
163154
IconButton(
164-
onClick = { viewModel.capturePhoto() },
155+
onClick = { viewModel.capturePhoto(onViewImage) },
165156
modifier = Modifier
166157
.size(72.dp)
167158
.background(Color.White.copy(alpha = 0.2f), CircleShape)

‎app/src/main/java/dev/androhit/natively/camera/ui/CameraViewModel.kt‎

Lines changed: 70 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
package dev.androhit.natively.camera.ui
22

3-
import androidx.camera.mlkit.vision.MlKitAnalyzer
3+
import android.graphics.Bitmap
44
import androidx.lifecycle.ViewModel
55
import androidx.lifecycle.viewModelScope
66
import dev.androhit.natively.camera.data.CameraController
7+
import dev.androhit.natively.data.TextAnalyzer
78
import dev.androhit.natively.domain.RecognizedText
9+
import dev.androhit.natively.domain.TextScript
810
import dev.androhit.natively.domain.TranslationRepository
911
import dev.androhit.natively.domain.models.Result
1012
import dev.androhit.natively.ui.components.CameraFeature
@@ -18,6 +20,7 @@ import kotlinx.coroutines.launch
1820
class CameraViewModel(
1921
private val cameraController: CameraController,
2022
private val translationRepository: TranslationRepository,
23+
private val textAnalyzer: TextAnalyzer,
2124
): ViewModel() {
2225

2326
private val _detectedTextLines = MutableStateFlow(emptyList<RecognizedText>())
@@ -32,16 +35,28 @@ class CameraViewModel(
3235
private val _translationState = MutableStateFlow(TranslationState())
3336
val translationState = _translationState.asStateFlow()
3437

38+
private val _capturedImage = MutableStateFlow<Bitmap?>(null)
39+
val capturedImage = _capturedImage.asStateFlow()
40+
41+
init {
42+
viewModelScope.launch {
43+
textAnalyzer.detectedTextLines.collect {
44+
_detectedTextLines.value = it
45+
}
46+
}
47+
}
48+
3549
fun onFeatureSelected(feature: CameraFeature) {
3650
_selectedFeature.value = feature
51+
_capturedImage.value = null
3752
if (feature == CameraFeature.ImageTranslate) {
3853
_detectedTextLines.value = emptyList()
3954
cameraController.clearAnalyzer()
4055
}
4156
}
4257

4358
fun translateText(text: String, detectedSource: String?) {
44-
_translationState.update { it.copy(isLoading = true) }
59+
_translationState.update { it.copy(isLoading = true, translatedText = null, error = null) }
4560
val targetLanguage = _translationState.value.targetLanguage.code
4661
val sourceLanguage = if(_translationState.value.sourceLanguage == Language.AUTO) {
4762
detectedSource
@@ -69,6 +84,18 @@ class CameraViewModel(
6984
}
7085
}
7186

87+
fun translateAllText() {
88+
_translationState.update { it.copy(isLoading = true) }
89+
val allText = _detectedTextLines.value.joinToString(" ") { it.text }
90+
if (allText.isBlank()) {
91+
_translationState.update { it.copy(error = "No text detected to translate", isLoading = false) }
92+
return
93+
}
94+
95+
val detectedSource = _detectedTextLines.value.firstOrNull()?.language
96+
translateText(allText, detectedSource)
97+
}
98+
7299
fun setSourceLanguage(language: Language) {
73100
_translationState.update { it.copy(sourceLanguage = language) }
74101
}
@@ -82,21 +109,52 @@ class CameraViewModel(
82109
_selectedTextLine.value = line
83110
}
84111

85-
fun switchCamera() {
86-
cameraController.switchCamera()
112+
fun analyzeCapturedImage() {
113+
_capturedImage.value?.let {
114+
viewModelScope.launch {
115+
val lines = textAnalyzer.analyzeImage(it)
116+
_detectedTextLines.value = lines
117+
}
118+
}
119+
}
120+
121+
fun attachTextAnalyzer() {
122+
cameraController.setAnalyzer(textAnalyzer.getInstance())
123+
}
124+
125+
fun updateScript(script: TextScript) {
126+
textAnalyzer.updateRecognizer(script)
127+
}
128+
129+
fun capturePhoto(onSuccess: () -> Unit) {
130+
cameraController.capturePhoto { result ->
131+
result.onSuccess { bitmap ->
132+
_capturedImage.value = bitmap
133+
onSuccess()
134+
}.onFailure {
135+
it.printStackTrace()
136+
}
137+
}
87138
}
88139

89-
fun attachTextAnalyzer(analyzer: MlKitAnalyzer) {
90-
cameraController.setAnalyzer(analyzer)
140+
fun clearCapturedImage() {
141+
_capturedImage.value = null
91142
}
92143

93-
fun onTextDetected(lines: List<RecognizedText>) {
94-
if (_selectedFeature.value == CameraFeature.LiveTranslate) {
95-
_detectedTextLines.value = lines
144+
fun cleanUp() {
145+
_translationState.update {
146+
it.copy(
147+
isLoading = false,
148+
translatedText = null,
149+
error = null
150+
)
96151
}
152+
cameraController.clearAnalyzer()
97153
}
98154

99-
fun capturePhoto() {
100-
/** TODO("Capture photo and store it") **/
155+
override fun onCleared() {
156+
super.onCleared()
157+
textAnalyzer.close()
158+
cameraController.detach()
101159
}
102-
}
160+
}

‎app/src/main/java/dev/androhit/natively/data/TextAnalyzer.kt‎

Lines changed: 49 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
11
package dev.androhit.natively.data
22

33
import android.content.Context
4+
import android.graphics.Bitmap
45
import androidx.camera.core.ImageAnalysis.COORDINATE_SYSTEM_VIEW_REFERENCED
56
import androidx.camera.mlkit.vision.MlKitAnalyzer
67
import androidx.core.content.ContextCompat
8+
import com.google.mlkit.vision.common.InputImage
9+
import com.google.mlkit.vision.text.Text
710
import com.google.mlkit.vision.text.TextRecognition
811
import com.google.mlkit.vision.text.chinese.ChineseTextRecognizerOptions
912
import com.google.mlkit.vision.text.devanagari.DevanagariTextRecognizerOptions
@@ -12,11 +15,17 @@ import com.google.mlkit.vision.text.korean.KoreanTextRecognizerOptions
1215
import com.google.mlkit.vision.text.latin.TextRecognizerOptions
1316
import dev.androhit.natively.domain.RecognizedText
1417
import dev.androhit.natively.domain.TextScript
18+
import kotlinx.coroutines.flow.MutableStateFlow
19+
import kotlinx.coroutines.flow.asStateFlow
20+
import kotlinx.coroutines.suspendCancellableCoroutine
21+
import kotlin.coroutines.resume
1522

1623
class TextAnalyzer(
17-
private val context: Context,
18-
private val onTextDetected: (List<RecognizedText>) -> Unit
24+
private val context: Context
1925
) {
26+
private val _detectedTextLines = MutableStateFlow<List<RecognizedText>>(emptyList())
27+
val detectedTextLines = _detectedTextLines.asStateFlow()
28+
2029
private val recognizers = mapOf(
2130
TextScript.Latin to TextRecognition.getClient(TextRecognizerOptions.DEFAULT_OPTIONS),
2231
TextScript.Devanagari to TextRecognition.getClient(DevanagariTextRecognizerOptions.Builder().build()),
@@ -35,26 +44,26 @@ class TextAnalyzer(
3544
ContextCompat.getMainExecutor(context)
3645
) { result ->
3746
val visionText = result?.getValue(recognizer) ?: return@MlKitAnalyzer
47+
val lines = visionText.toRecognizedText()
48+
_detectedTextLines.value = lines
49+
}
50+
}
3851

39-
val lines = visionText.textBlocks.flatMap { block ->
40-
block.lines.mapNotNull { line ->
41-
val confidence = line.elements
42-
.mapNotNull { it.confidence }
43-
.average()
44-
.toFloat()
45-
46-
if (confidence < MAX_CONFIDENCE) return@mapNotNull null
52+
suspend fun analyzeImage(bitmap: Bitmap, script: TextScript = TextScript.Latin): List<RecognizedText> {
53+
val inputImage = InputImage.fromBitmap(bitmap, 0)
54+
val recognizer = recognizers.getValue(script)
4755

48-
line.boundingBox?.let { rect ->
49-
RecognizedText(
50-
text = line.text,
51-
boundingBox = rect,
52-
language = line.recognizedLanguage
53-
)
54-
}
56+
return suspendCancellableCoroutine { continuation ->
57+
recognizer.process(inputImage)
58+
.addOnSuccessListener { visionText ->
59+
val lines = visionText.toRecognizedText()
60+
_detectedTextLines.value = lines
61+
continuation.resume(lines)
62+
}
63+
.addOnFailureListener { e ->
64+
e.printStackTrace()
65+
continuation.resume(emptyList())
5566
}
56-
}
57-
onTextDetected(lines)
5867
}
5968
}
6069

@@ -66,6 +75,27 @@ class TextAnalyzer(
6675
recognizers.values.forEach { it.close() }
6776
}
6877

78+
private fun Text.toRecognizedText(): List<RecognizedText> {
79+
return this.textBlocks.flatMap { block ->
80+
block.lines.mapNotNull { line ->
81+
val confidence = line.elements
82+
.mapNotNull { it.confidence }
83+
.average()
84+
.toFloat()
85+
86+
if (confidence < MAX_CONFIDENCE) return@mapNotNull null
87+
88+
line.boundingBox?.let { rect ->
89+
RecognizedText(
90+
text = line.text,
91+
boundingBox = rect,
92+
language = line.recognizedLanguage
93+
)
94+
}
95+
}
96+
}
97+
}
98+
6999
companion object {
70100
private const val MAX_CONFIDENCE = 0.7f
71101
}

‎app/src/main/java/dev/androhit/natively/di/AppModule.kt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,13 @@ package dev.androhit.natively.di
22

33
import dev.androhit.natively.camera.data.CameraController
44
import dev.androhit.natively.camera.ui.CameraViewModel
5+
import dev.androhit.natively.data.TextAnalyzer
56
import org.koin.core.module.dsl.singleOf
67
import org.koin.core.module.dsl.viewModelOf
78
import org.koin.dsl.module
89

910
val appModule = module {
11+
singleOf(::TextAnalyzer)
1012
singleOf(::CameraController)
1113
viewModelOf(::CameraViewModel)
1214
}

0 commit comments

Comments
 (0)