CameraActivity with Google ML KIT for decoding QR codes

client/android/AndroidManifest.xml

@@ -114,6 +114,10 @@
 
         </activity>
     
+        <activity
+            android:name=".CameraActivity"
+            android:exported="false" />
+
         <service
             android:name=".VPNService"
             android:process=":QtOnlyProcess"

client/android/build.gradle

@@ -2,7 +2,7 @@ apply plugin: 'com.github.ben-manes.versions'
 
 buildscript {
    ext{
-       kotlin_version = "1.4.30-M1"
+       kotlin_version = "1.7.22"
        //  for libwg
        appcompatVersion = '1.1.0'
        annotationsVersion = '1.0.1'
@@ -28,12 +28,6 @@ buildscript {
     }
 }
 
-repositories {
-    google()
-    jcenter()
-    mavenCentral()
-}
-
 apply plugin: 'com.android.application'
 apply plugin: 'kotlin-android'
 apply plugin: 'kotlin-android-extensions'
@@ -42,14 +36,34 @@ apply plugin: 'kotlin-kapt'
 
 dependencies {
     implementation fileTree(dir: 'libs', include: ['*.jar', '*.aar'])
-    implementation 'androidx.core:core-ktx:1.1.0'
+
+    implementation 'androidx.core:core-ktx:1.7.0'
+
+    implementation 'androidx.appcompat:appcompat:1.4.1'
+
     implementation "androidx.security:security-crypto:1.1.0-alpha03"
     implementation "androidx.security:security-identity-credential:1.0.0-alpha02"
+
     implementation "org.jetbrains.kotlinx:kotlinx-serialization-json:1.2.2"
     implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:1.3.0"
     implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.0"
+
     coreLibraryDesugaring "com.android.tools:desugar_jdk_libs:1.1.5"
+
     implementation project(path: ':shadowsocks')
+
+    // CameraX core library using the camera2 implementation
+    def camerax_version = "1.2.1"
+    implementation("androidx.camera:camera-core:${camerax_version}")
+    implementation("androidx.camera:camera-camera2:${camerax_version}")
+    implementation("androidx.camera:camera-lifecycle:${camerax_version}")
+    implementation("androidx.camera:camera-view:${camerax_version}")
+    implementation("androidx.camera:camera-extensions:${camerax_version}")
+
+    def camerax_ml_version = "1.2.0-beta02"
+    def ml_kit_version = "17.0.3"
+    implementation("androidx.camera:camera-mlkit-vision:${camerax_ml_version}")
+    implementation("com.google.mlkit:barcode-scanning:${ml_kit_version}")
 }
 
 androidExtensions {

client/android/res/layout/activity_camera.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="utf-8"?>
+<androidx.constraintlayout.widget.ConstraintLayout xmlns:android="http://schemas.android.com/apk/res/android"
+	xmlns:app="http://schemas.android.com/apk/res-auto"
+	xmlns:tools="http://schemas.android.com/tools"
+	android:layout_width="match_parent"
+	android:layout_height="match_parent"
+	tools:context=".CameraActivity">
+	
+	<androidx.camera.view.PreviewView
+		android:id="@+id/viewFinder"
+		android:layout_width="match_parent"
+		android:layout_height="match_parent" />
+
+</androidx.constraintlayout.widget.ConstraintLayout>

client/android/settings.gradle

@@ -1 +1,19 @@
+pluginManagement {
+    repositories {
+        google()
+        mavenCentral()
+        jcenter()
+        gradlePluginPortal()
+    }
+}
+
+dependencyResolutionManagement {
+    repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS)
+    repositories {
+        google()
+        mavenCentral()
+        jcenter()
+    }
+}
+
 include ':shadowsocks'

client/android/shadowsocks/build.gradle

@@ -1,21 +1,9 @@
-
-allprojects {
-    repositories {
-        google()
-        jcenter()
-        mavenCentral()
-    }
-}
-
-
 apply plugin: 'com.android.library'
 apply plugin: 'kotlin-android'
 apply plugin: 'kotlin-android-extensions'
 apply plugin: 'kotlin-kapt'
 //apply plugin: 'com.novoda.bintray-release'
 
-
-
 android {
     compileSdkVersion 30
     defaultConfig {

client/android/src/org/amnezia/vpn/qt/CameraActivity.kt

@@ -0,0 +1,150 @@
+package org.amnezia.vpn.qt
+
+import android.Manifest
+import android.annotation.SuppressLint
+import android.app.Activity
+import android.content.Intent
+import android.content.pm.PackageManager
+import android.os.Bundle
+import android.util.Log
+import android.widget.Toast
+import androidx.appcompat.app.AppCompatActivity
+import androidx.camera.core.*
+import androidx.camera.lifecycle.ProcessCameraProvider
+import androidx.camera.view.PreviewView
+import androidx.core.content.ContextCompat
+import com.google.mlkit.vision.barcode.BarcodeScannerOptions
+import com.google.mlkit.vision.barcode.BarcodeScanning
+import com.google.mlkit.vision.barcode.common.Barcode
+import com.google.mlkit.vision.common.InputImage
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+import org.amnezia.vpn.R
+
+
+class CameraActivity : AppCompatActivity() {
+
+    private val CAMERA_REQUEST = 100
+
+    private lateinit var cameraExecutor: ExecutorService
+    private lateinit var analyzerExecutor: ExecutorService
+
+    private lateinit var viewFinder: PreviewView
+
+    override fun onCreate(savedInstanceState: Bundle?) {
+        super.onCreate(savedInstanceState)
+        setContentView(R.layout.activity_camera)
+
+        viewFinder = findViewById(R.id.viewFinder)
+
+        cameraExecutor = Executors.newSingleThreadExecutor()
+        analyzerExecutor = Executors.newSingleThreadExecutor()
+
+        checkPermissions()
+
+        configureVideoPreview()
+    }
+
+    private fun checkPermissions() {
+        if (checkSelfPermission(Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) {
+            requestPermissions(arrayOf(Manifest.permission.CAMERA), CAMERA_REQUEST)
+        }
+    }
+
+    override fun onRequestPermissionsResult(requestCode: Int, permissions: Array<out String>, grantResults: IntArray) {
+        super.onRequestPermissionsResult(requestCode, permissions, grantResults)
+        if (requestCode == CAMERA_REQUEST) {
+            if (grantResults[0] == PackageManager.PERMISSION_GRANTED) {
+                Toast.makeText(this, "CameraX permission granted", Toast.LENGTH_SHORT).show();
+            } else {
+                Toast.makeText(this, "CameraX permission denied", Toast.LENGTH_SHORT).show();
+            }
+        }
+    }
+
+    @SuppressLint("UnsafeOptInUsageError")
+    private fun configureVideoPreview() {
+        val cameraProviderFuture = ProcessCameraProvider.getInstance(this)
+        val imageCapture = ImageCapture.Builder().build()
+
+        cameraProviderFuture.addListener({
+            val cameraProvider: ProcessCameraProvider = cameraProviderFuture.get()
+
+            val preview = Preview.Builder().build()
+
+            val cameraSelector = CameraSelector.DEFAULT_BACK_CAMERA
+
+            val listener = object : OnBarcodeActivityResult {
+                override fun onSuccess(result: String) {
+                    val resultIntent = Intent()
+                    resultIntent.putExtra("result", result)
+                    setResult(Activity.RESULT_OK, resultIntent)
+                    finish()
+                }
+
+                override fun onFailure(result: Exception) {
+                    Log.d("WUTT", "exception: $result")
+                }
+            }
+
+            val imageAnalyzer = BarCodeAnalyzer(listener)
+
+            val analysisUseCase = ImageAnalysis.Builder()
+                .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
+                .build()
+
+            analysisUseCase.setAnalyzer(analyzerExecutor, imageAnalyzer)
+
+            try {
+                preview.setSurfaceProvider(viewFinder.surfaceProvider)
+                cameraProvider.unbindAll()
+                cameraProvider.bindToLifecycle(this, cameraSelector, preview, imageCapture, analysisUseCase)
+            } catch(exc: Exception) {
+                Log.e("WUTT", "Use case binding failed", exc)
+            }
+        }, ContextCompat.getMainExecutor(this))
+    }
+
+    override fun onDestroy() {
+        cameraExecutor.shutdown()
+        analyzerExecutor.shutdown()
+
+        super.onDestroy()
+    }
+
+    private class BarCodeAnalyzer(val callback: OnBarcodeActivityResult): ImageAnalysis.Analyzer {
+
+        private val options = BarcodeScannerOptions.Builder()
+            .setBarcodeFormats(Barcode.FORMAT_QR_CODE)
+            .build()
+
+        private val scanner = BarcodeScanning.getClient(options)
+
+        @SuppressLint("UnsafeOptInUsageError")
+        override fun analyze(imageProxy: ImageProxy) {
+            val mediaImage = imageProxy.image
+
+            if (mediaImage != null) {
+                val image = InputImage.fromMediaImage(mediaImage, imageProxy.imageInfo.rotationDegrees)
+
+                scanner.process(image)
+                    .addOnSuccessListener { barcodes ->
+                        if (barcodes.isNotEmpty()) {
+                            callback.onSuccess(barcodes[0]?.displayValue ?: "empty?")
+                        }
+
+                        imageProxy.close()
+                    }
+                    .addOnFailureListener {
+                        callback.onFailure(it)
+                        imageProxy.close()
+                    }
+            }
+        }
+    }
+
+    interface OnBarcodeActivityResult {
+        fun onSuccess(result: String)
+        fun onFailure(result: Exception)
+    }
+}

client/android/src/org/amnezia/vpn/qt/VPNActivity.kt

@@ -36,6 +36,8 @@ class VPNActivity : org.qtproject.qt.android.bindings.QtActivity() {
     private val TAG = "VPNActivity"
     private val STORAGE_PERMISSION_CODE = 42
 
+    private val CAMERA_ACTION_CODE = 101
+
     companion object {
         private lateinit var instance: VPNActivity
 
@@ -47,6 +49,10 @@ class VPNActivity : org.qtproject.qt.android.bindings.QtActivity() {
             VPNActivity.getInstance().initServiceConnection()
         }
 
+        @JvmStatic fun startQrCodeReader() {
+            VPNActivity.getInstance().startQrCodeActivity()
+        }
+
         @JvmStatic fun sendToService(actionCode: Int, body: String) {
             VPNActivity.getInstance().dispatchParcel(actionCode, body)
         }
@@ -62,7 +68,12 @@ class VPNActivity : org.qtproject.qt.android.bindings.QtActivity() {
 
         super.onCreate(savedInstanceState)
 
-        instance = this;
+        instance = this
+    }
+
+    private fun startQrCodeActivity() {
+        val intent = Intent(this, CameraActivity::class.java)
+        startActivityForResult(intent, CAMERA_ACTION_CODE)
     }
 
     override fun getSystemService(name: String): Any? {
@@ -82,6 +93,7 @@ class VPNActivity : org.qtproject.qt.android.bindings.QtActivity() {
     external fun onServiceMessage(actionCode: Int, body: String?)
     external fun qtOnServiceConnected()
     external fun qtOnServiceDisconnected()
+    external fun onActivityMessage(actionCode: Int, body: String?)
 
     private fun dispatchParcel(actionCode: Int, body: String) {
         if (!isBound) {
@@ -286,6 +298,7 @@ class VPNActivity : org.qtproject.qt.android.bindings.QtActivity() {
     private val EVENT_PERMISSION_REQURED = 6
     private val EVENT_DISCONNECTED = 2
 
+    private val UI_EVENT_QR_CODE_RECEIVED = 0
 
     fun onPermissionRequest(code: Int, data: Parcel?) {
         if (code != EVENT_PERMISSION_REQURED) {
@@ -310,7 +323,13 @@ class VPNActivity : org.qtproject.qt.android.bindings.QtActivity() {
             }
             return
         }
+
         super.onActivityResult(requestCode, resultCode, data)
+
+        if (requestCode == CAMERA_ACTION_CODE && resultCode == RESULT_OK) {
+            val extra = data?.getStringExtra("result") ?: ""
+            onActivityMessage(UI_EVENT_QR_CODE_RECEIVED, extra)
+        }
     }
 
     override fun onKeyDown(keyCode: Int, event: KeyEvent): Boolean {

client/platforms/android/androidvpnactivity.cpp

@@ -22,12 +22,10 @@ AndroidVPNActivity::AndroidVPNActivity() {
     AndroidUtils::runOnAndroidThreadAsync([]() {
         JNINativeMethod methods[]{
             {"handleBackButton", "()Z", reinterpret_cast<bool*>(handleBackButton)},
-            {"onServiceMessage", "(ILjava/lang/String;)V",
+            {"onServiceMessage", "(ILjava/lang/String;)V", reinterpret_cast<void*>(onServiceMessage)},
-             reinterpret_cast<void*>(onServiceMessage)},
+            {"qtOnServiceConnected", "()V", reinterpret_cast<void*>(onServiceConnected)},
-            {"qtOnServiceConnected", "()V",
+            {"qtOnServiceDisconnected", "()V", reinterpret_cast<void*>(onServiceDisconnected)},
-             reinterpret_cast<void*>(onServiceConnected)},
+            {"onActivityMessage", "(ILjava/lang/String;)V", reinterpret_cast<void*>(onAndroidVpnActivityMessage)}
-            {"qtOnServiceDisconnected", "()V",
-             reinterpret_cast<void*>(onServiceDisconnected)},
         };
 
         QJniObject javaClass(CLASSNAME);
@@ -54,6 +52,11 @@ void AndroidVPNActivity::connectService() {
     QJniObject::callStaticMethod<void>(CLASSNAME, "connectService", "()V");
 }
 
+void AndroidVPNActivity::startQrCodeReader()
+{
+    QJniObject::callStaticMethod<void>(CLASSNAME, "startQrCodeReader", "()V");
+}
+
 // static
 AndroidVPNActivity* AndroidVPNActivity::instance() {
     if (s_instance == nullptr) {
@@ -121,6 +124,19 @@ void AndroidVPNActivity::handleServiceMessage(int code, const QString& data) {
     }
 }
 
+void AndroidVPNActivity::handleActivityMessage(int code, const QString &data)
+{
+    auto mode = (UIEvents)code;
+
+    switch (mode) {
+    case UIEvents::QR_CODED_DECODED:
+        emit eventQrCodeReceived(data);
+        break;
+    default:
+        Q_ASSERT(false);
+    }
+}
+
 void AndroidVPNActivity::onServiceConnected(JNIEnv* env, jobject thiz) {
     Q_UNUSED(env);
     Q_UNUSED(thiz);
@@ -134,3 +150,19 @@ void AndroidVPNActivity::onServiceDisconnected(JNIEnv* env, jobject thiz) {
 
     emit AndroidVPNActivity::instance()->serviceDisconnected();
 }
+
+void AndroidVPNActivity::onAndroidVpnActivityMessage(JNIEnv *env, jobject thiz, jint messageType, jstring message)
+{
+    Q_UNUSED(thiz);
+    const char* buffer = env->GetStringUTFChars(message, nullptr);
+    if (!buffer) {
+        return;
+    }
+
+    QString parcelBody(buffer);
+    env->ReleaseStringUTFChars(message, buffer);
+
+    AndroidUtils::dispatchToMainThread([messageType, parcelBody] {
+        AndroidVPNActivity::instance()->handleActivityMessage(messageType, parcelBody);
+    });
+}

client/platforms/android/androidvpnactivity.h

@@ -59,6 +59,11 @@ enum ServiceEvents {
 };
 typedef enum ServiceEvents ServiceEvents;
 
+enum UIEvents {
+    QR_CODED_DECODED = 0,
+};
+typedef enum UIEvents UIEvents;
+
 class AndroidVPNActivity : public QObject
 {
     Q_OBJECT
@@ -69,6 +74,7 @@ public:
     static bool handleBackButton(JNIEnv* env, jobject thiz);
     static void sendToService(ServiceAction type, const QString& data);
     static void connectService();
+    static void startQrCodeReader();
 
 signals:
     void serviceConnected();
@@ -80,6 +86,7 @@ signals:
     void eventBackendLogs(const QString& data);
     void eventActivationError(const QString& data);
     void eventConfigImport(const QString& data);
+    void eventQrCodeReceived(const QString& data);
 
 private:
     AndroidVPNActivity();
@@ -87,7 +94,9 @@ private:
     static void onServiceMessage(JNIEnv* env, jobject thiz, jint messageType, jstring body);
     static void onServiceConnected(JNIEnv* env, jobject thiz);
     static void onServiceDisconnected(JNIEnv* env, jobject thiz);
+    static void onAndroidVpnActivityMessage(JNIEnv* env, jobject thiz, jint messageType, jstring message);
     void handleServiceMessage(int code, const QString& data);
+    void handleActivityMessage(int code, const QString& data);
 };
 
 #endif // ANDROIDVPNACTIVITY_H