From ad01fd5671bc74355b14fe257cab5e0b6ce1731d Mon Sep 17 00:00:00 2001 From: RyuJongJin <266630912+RyuJongJin@users.noreply.github.com> Date: Sun, 4 Oct 2026 20:18:27 +0900 Subject: [PATCH] =?UTF-8?q?Android=20=EC=9E=91=EC=97=85=20=EC=95=8C?= =?UTF-8?q?=EB=A6=BC=EC=9D=84=20=EB=88=84=EB=A5=B4=EB=A9=B4=20=EC=9E=91?= =?UTF-8?q?=EC=97=85=20=ED=98=84=ED=99=A9=20=ED=99=94=EB=A9=B4,=20?= =?UTF-8?q?=EC=9C=84=EC=AA=BD=20=EC=9E=91=EC=97=85=20=ED=91=9C=EC=8B=9C?= =?UTF-8?q?=EC=97=90=20=EC=A7=84=ED=96=89=EB=A5=A0=20=C2=B7=20=EB=88=84?= =?UTF-8?q?=EB=A5=B4=EB=A9=B4=20=EC=9E=91=EC=97=85=20=ED=98=84=ED=99=A9,?= =?UTF-8?q?=20=EC=95=B1=20=ED=99=94=EB=A9=B4=EC=9D=80=20=ED=95=98=EB=82=98?= =?UTF-8?q?=EB=A7=8C=20(singleTask).=20AI=20=EC=9D=8C=EC=84=B1=EC=9D=B8?= =?UTF-8?q?=EC=8B=9D=20=EC=A4=91=20=EC=B7=A8=EC=86=8C=EA=B0=80=20=EB=B0=94?= =?UTF-8?q?=EB=A1=9C=20=EB=A8=B9=ED=9E=88=EA=B2=8C=20(whisper=20abort,=20A?= =?UTF-8?q?ndroid=20=ED=83=9C=EA=B7=B8=20=ED=8F=AC=EC=9D=B8=ED=84=B0)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- android/app/src/main/AndroidManifest.xml | 2 +- .../com/jj/jj_mkvmaker/KeepAliveService.kt | 5 +- .../kotlin/com/jj/jj_mkvmaker/MainActivity.kt | 17 ++++ integration_test/ai_cancel_test.dart | 39 ++++++++ lib/app/app_controller.dart | 8 +- lib/main.dart | 11 +++ lib/platform/common/whisper_recognizer.dart | 51 ++++++++--- lib/services/ai_services.dart | 3 + lib/ui/work_panel.dart | 89 ++++++++++++++++++- packages/whisper_ggml/JJ_PATCH.md | 5 ++ .../whisper_ggml/android/src/whisper/main.cpp | 19 ++++ packages/whisper_ggml/lib/src/whisper.dart | 12 ++- test/ai_test.dart | 2 + test/batch_test.dart | 2 + 14 files changed, 246 insertions(+), 19 deletions(-) create mode 100644 integration_test/ai_cancel_test.dart diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index af27a09..9cd7039 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -24,7 +24,7 @@ = Build.VERSION_CODES.O) { diff --git a/android/app/src/main/kotlin/com/jj/jj_mkvmaker/MainActivity.kt b/android/app/src/main/kotlin/com/jj/jj_mkvmaker/MainActivity.kt index 7e07c88..ebe23d3 100644 --- a/android/app/src/main/kotlin/com/jj/jj_mkvmaker/MainActivity.kt +++ b/android/app/src/main/kotlin/com/jj/jj_mkvmaker/MainActivity.kt @@ -17,14 +17,23 @@ import java.io.File /// 저장소 전체 접근 (동영상 옆 jj_mkv 폴더에 MKV 를 만들기 위해) 확인 · 요청, 다운로드 프로그램 준비 class MainActivity : FlutterActivity() { + companion object { + /// 작업 알림을 눌렀을 때: 작업 현황 화면을 연다 + const val ACTION_SHOW_JOBS = "com.jj.jj_mkvmaker.SHOW_JOBS" + } + private var channel: MethodChannel? = null + /// 앱이 켜질 때 작업 알림으로 열렸는지 (Dart 가 준비되면 가져간다) + private var pendingShowJobs = false + /// 다른 앱에서 연 동영상 (앱이 켜질 때 받은 것: Dart 가 준비되면 가져간다) private var pendingOpen: List> = emptyList() override fun configureFlutterEngine(flutterEngine: FlutterEngine) { super.configureFlutterEngine(flutterEngine) pendingOpen = openedFiles(intent) + pendingShowJobs = intent?.action == ACTION_SHOW_JOBS val ch = MethodChannel(flutterEngine.dartExecutor.binaryMessenger, "jj_mkvmaker/android") channel = ch ch.setMethodCallHandler { call, result -> @@ -33,6 +42,10 @@ class MainActivity : FlutterActivity() { result.success(pendingOpen) pendingOpen = emptyList() } + "takeShowJobs" -> { + result.success(pendingShowJobs) + pendingShowJobs = false + } "hasAllFilesAccess" -> result.success(hasAllFilesAccess()) "requestAllFilesAccess" -> { requestAllFilesAccess() @@ -98,6 +111,10 @@ class MainActivity : FlutterActivity() { /// 앱이 켜져 있을 때 다른 앱에서 동영상을 열면 (launchMode singleTop) override fun onNewIntent(intent: Intent) { super.onNewIntent(intent) + if (intent.action == ACTION_SHOW_JOBS) { + channel?.invokeMethod("showJobs", null) + return + } val files = openedFiles(intent) if (files.isNotEmpty()) channel?.invokeMethod("openFiles", files) } diff --git a/integration_test/ai_cancel_test.dart b/integration_test/ai_cancel_test.dart new file mode 100644 index 0000000..2eac392 --- /dev/null +++ b/integration_test/ai_cancel_test.dart @@ -0,0 +1,39 @@ +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:integration_test/integration_test.dart'; +import 'package:jj_mkvmaker/app/app_controller.dart'; +import 'package:jj_mkvmaker/core/languages.dart'; +import 'package:jj_mkvmaker/core/models.dart'; +import 'package:jj_mkvmaker/services/model_store.dart'; +import 'package:jj_mkvmaker/services/platform_services.dart'; + +/// 긴 영상 (JJ_LONG_VIDEO) 의 음성인식 도중 [작업 취소] → 몇 초 안에 멈춘다 (끝까지 기다리지 않음) +void main() { + IntegrationTestWidgetsFlutterBinding.ensureInitialized(); + + testWidgets('음성인식 중 취소', (tester) async { + final video = Platform.environment['JJ_LONG_VIDEO']; + if (video == null || !File(video).existsSync()) return markTestSkipped('JJ_LONG_VIDEO 없음'); + final c = AppController(PlatformServices.create()); + await tester.runAsync(() async { + await c.init(); + await c.addVideos([video]); + final v = c.videos.single; + final job = c.generateAiSubtitles([v], AiOptions(targets: {languageOf('en')}, whisper: whisperModels.first)); + // 음성인식이 시작될 때까지 (음성 추출이 끝나고) + for (var i = 0; i < 600 && !(v.phase ?? '').startsWith('음성인식'); i++) { + await Future.delayed(const Duration(milliseconds: 100)); + } + await Future.delayed(const Duration(seconds: 5)); + final sw = Stopwatch()..start(); + c.cancel(); + await job; + // ignore: avoid_print + print('RESULT 취소 후 멈추기까지 ${sw.elapsedMilliseconds}ms · 상태 ${v.status} · ${c.logs.last}'); + expect(sw.elapsed.inSeconds, lessThan(15)); + expect(v.status, JobStatus.ready); + expect(c.logs.last, contains('취소')); + }); + }, timeout: const Timeout(Duration(minutes: 10))); +} diff --git a/lib/app/app_controller.dart b/lib/app/app_controller.dart index 7bc685c..b2c5bc0 100644 --- a/lib/app/app_controller.dart +++ b/lib/app/app_controller.dart @@ -825,7 +825,9 @@ class AppController extends ChangeNotifier { // 3. 음성인식 (5% ~ 50%) phase(tr('음성인식 중'), 0.05); - final raw = await services.createRecognizer!().transcribe( + final recognizer = _recognizer = services.createRecognizer!(); + if (_aiCancelled) throw const AiCancelled(); + final raw = await recognizer.transcribe( wav, modelPath: await services.models.pathOf(opts.whisper), language: opts.source == undetermined ? 'auto' : whisperCode(opts.source), @@ -1279,7 +1281,11 @@ class AppController extends ChangeNotifier { static String get missingFileMessage => tr('동영상 파일이 없습니다 (옮겨졌거나 지워졌습니다). 목록에서 빼고 다시 추가하세요.'); /// 지금 작업 중단 + 대기 중인 작업 모두 비우기 + /// 지금 음성인식 (취소하면 멈춘다) + SpeechRecognizer? _recognizer; + void cancel() { + _recognizer?.cancel(); if (pendingJobs.isNotEmpty) _log(trf('대기 중인 작업 {0}개 취소', [pendingJobs.length])); pendingJobs.clear(); for (final (_, done) in _pendingRuns) { diff --git a/lib/main.dart b/lib/main.dart index 78a49f4..c044eb0 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -38,6 +38,7 @@ import 'ui/player_page.dart'; import 'ui/setup_dialog.dart'; import 'ui/update_dialog.dart'; import 'ui/theme.dart'; +import 'ui/work_panel.dart'; import 'l10n/tr.dart'; import 'app/i18n_controller.dart'; @@ -408,6 +409,12 @@ Future runAndroid(String dataDir) async { // 연결 프로그램 · 공유로 받은 동영상: 앱이 켜져 있을 때 (MainActivity.onNewIntent) const android = MethodChannel('jj_mkvmaker/android'); android.setMethodCallHandler((call) async { + if (call.method == 'showJobs') { + // 작업 알림을 눌렀다: 작업 현황 화면으로 + final nav = navigatorKey.currentState; + if (nav != null) unawaited(WorkStatusPage.open(nav, controller, downloads)); + return null; + } if (call.method == 'openFiles') { await openFromOtherApp(navigatorKey, controller, (call.arguments as List).cast>()); } @@ -417,6 +424,10 @@ Future runAndroid(String dataDir) async { WidgetsBinding.instance.addPostFrameCallback((_) async { // 연결 프로그램 · 공유로 앱을 켠 경우: 그 동영상부터 try { + if (await android.invokeMethod('takeShowJobs') ?? false) { + final nav = navigatorKey.currentState; + if (nav != null) unawaited(WorkStatusPage.open(nav, controller, downloads)); + } final opened = await android.invokeMethod>('takeOpenedFiles') ?? const []; if (opened.isNotEmpty) { unawaited(openFromOtherApp(navigatorKey, controller, opened.cast>())); diff --git a/lib/platform/common/whisper_recognizer.dart b/lib/platform/common/whisper_recognizer.dart index 0e57d50..a7939a1 100644 --- a/lib/platform/common/whisper_recognizer.dart +++ b/lib/platform/common/whisper_recognizer.dart @@ -1,6 +1,8 @@ +import 'dart:ffi' as ffi; import 'dart:io'; import 'dart:math' as math; +import 'package:ffi/ffi.dart'; import 'package:whisper_ggml/whisper_ggml.dart'; import '../../core/srt.dart'; @@ -8,6 +10,15 @@ import '../../services/ai_services.dart'; /// whisper.cpp 음성인식. Windows·Android 공용. class WhisperRecognizer implements SpeechRecognizer { + /// 지금 음성인식의 중단 신호 (네이티브가 읽는 int32). 0 이 아니면 멈춘다. + ffi.Pointer? _abort; + + @override + void cancel() { + final a = _abort; + if (a != null) a.value = 1; + } + @override Future> transcribe( String wavPath, { @@ -17,19 +28,31 @@ class WhisperRecognizer implements SpeechRecognizer { }) async { // 실제 코어 수 (하이퍼스레딩 절반) 가 가장 빠름 final threads = math.max(1, math.min(8, Platform.numberOfProcessors ~/ 2)); - final res = await const Whisper(model: WhisperModel.base).transcribe( - transcribeRequest: TranscribeRequest( - audio: wavPath, - language: language, - threads: threads, - suppressNonSpeechTokens: true, - ), - modelPath: modelPath, - onProgress: onProgress == null ? null : (pct) => onProgress(pct / 100), - ); - return [ - for (final s in res.segments ?? const []) - Cue(s.fromTs, s.toTs, s.text), - ]; + final abort = _abort = calloc(); + try { + final res = await const Whisper(model: WhisperModel.base).transcribe( + transcribeRequest: TranscribeRequest( + audio: wavPath, + language: language, + threads: threads, + suppressNonSpeechTokens: true, + ), + modelPath: modelPath, + onProgress: onProgress == null ? null : (pct) => onProgress(pct / 100), + abortFlagAddress: abort.address, + ); + if (abort.value != 0) throw const AiCancelled(); + return [ + for (final s in res.segments ?? const []) + Cue(s.fromTs, s.toTs, s.text), + ]; + } catch (e) { + // 취소로 멈춘 것이면 "취소" 로 (네이티브는 "failed to process audio" 오류를 낸다) + if (abort.value != 0) throw const AiCancelled(); + rethrow; + } finally { + _abort = null; + calloc.free(abort); + } } } diff --git a/lib/services/ai_services.dart b/lib/services/ai_services.dart index 031772b..ca2b1fc 100644 --- a/lib/services/ai_services.dart +++ b/lib/services/ai_services.dart @@ -19,6 +19,9 @@ abstract class SpeechRecognizer { String language = 'auto', AiProgress? onProgress, }); + + /// 진행 중인 음성인식 멈추기 (transcribe 가 곧 오류로 끝난다) + void cancel(); } /// 번역 경계. 구현: platform/common/nllb_translator.dart (Windows·Android 공용) diff --git a/lib/ui/work_panel.dart b/lib/ui/work_panel.dart index ec49dbe..2f1251f 100644 --- a/lib/ui/work_panel.dart +++ b/lib/ui/work_panel.dart @@ -4,10 +4,83 @@ import '../app/app_controller.dart'; import '../app/download_manager.dart'; import '../core/models.dart'; import '../services/downloader.dart'; +import 'app_actions.dart'; import 'downloads_page.dart'; import 'theme.dart'; import '../l10n/tr.dart'; +/// "작업 현황" 화면 (전체 화면): Android 작업 알림 · 위쪽 작업 표시를 누르면 연다 +class WorkStatusPage extends StatefulWidget { + final AppController c; + final DownloadManager? downloads; + const WorkStatusPage({super.key, required this.c, this.downloads}); + + /// 열려 있는 작업 현황 화면 수 + static int _open = 0; + + @override + State createState() => _WorkStatusPageState(); + + static const routeName = 'jobs'; + + /// 작업 현황으로: 이미 열려 있으면 그 화면으로, 없으면 새로 + static Future open(NavigatorState nav, AppController c, DownloadManager? downloads) async { + if (_open > 0) { + var found = false; + nav.popUntil((r) { + if (r.settings.name == routeName) found = true; + return found || r.isFirst; + }); + if (found) return; + } + await nav.push(MaterialPageRoute( + settings: const RouteSettings(name: routeName), + builder: (_) => WorkStatusPage(c: c, downloads: downloads), + )); + } + +} + +class _WorkStatusPageState extends State { + @override + void initState() { + super.initState(); + WorkStatusPage._open++; + } + + @override + void dispose() { + WorkStatusPage._open--; + super.dispose(); + } + + @override + Widget build(BuildContext context) => Scaffold( + body: Column(children: [ + Container( + height: appBarHeight, + color: JjColors.panel, + padding: const EdgeInsets.only(left: 8, right: appBarRightPadding), + child: Row(children: [ + const AppNavButtons(), + const SizedBox(width: 8), + Text(tr('작업 현황'), style: const TextStyle(fontSize: 16, fontWeight: FontWeight.w600)), + const Spacer(), + AppActions(c: widget.c), + ]), + ), + Expanded( + child: WorkPanel( + c: widget.c, + downloads: widget.downloads, + onClose: () => Navigator.maybePop(context), + onOpenHome: () => AppNavButtons.toMkv(context), + ), + ), + ]), + ); +} + /// 브라우저 오른쪽 "작업 현황" (화면 분할): 자막 · MKV 작업과 다운로드 진행을 보면서 인터넷을 볼 수 있게. class WorkPanel extends StatelessWidget { final AppController c; @@ -308,7 +381,10 @@ class JobIndicator extends StatelessWidget { : running.map((v) => v.progress).reduce((a, b) => a + b) / running.length; final wait = c.pendingJobs.length; - return Tooltip( + // 누르면 작업 현황 화면 (어느 화면에서나) + final scope = AppScope.maybeOf(context); + final nav = Navigator.maybeOf(context); + final body = Tooltip( message: [ trf('지금: {0}', [c.currentJob ?? '']), for (final v in running) @@ -327,6 +403,11 @@ class JobIndicator extends StatelessWidget { value: p == null || p == 0 ? null : p, ), ), + // 좁아서 아이콘만일 때도 진행률은 보이게 + if (iconOnly && p != null) ...[ + const SizedBox(width: 4), + Text('${(p * 100).round()}%', style: const TextStyle(fontSize: 12, color: JjColors.accent)), + ], if (!iconOnly) const SizedBox(width: 6), if (!iconOnly) ConstrainedBox( @@ -341,5 +422,11 @@ class JobIndicator extends StatelessWidget { ], ), ); + if (scope == null || nav == null) return body; + return InkWell( + borderRadius: BorderRadius.circular(6), + onTap: () => WorkStatusPage.open(nav, scope.controller, scope.downloads), + child: Padding(padding: const EdgeInsets.symmetric(horizontal: 4, vertical: 6), child: body), + ); } } diff --git a/packages/whisper_ggml/JJ_PATCH.md b/packages/whisper_ggml/JJ_PATCH.md index 0dfa68c..c21170c 100644 --- a/packages/whisper_ggml/JJ_PATCH.md +++ b/packages/whisper_ggml/JJ_PATCH.md @@ -19,3 +19,8 @@ Windows 에서 음성인식이 끝나는 순간 앱이 종료됨 (예외 코드 - pubspec.yaml · lib/src/whisper_audio_convert.dart: `ffmpeg_kit_flutter_new_min` → `ffmpeg_kit_flutter_new_min_gpl` (앱이 MKV 만들기에 쓰는 것과 같은 것. 두 가지가 함께 들어가면 같은 이름의 Java 클래스 · 네이티브 라이브러리가 겹쳐 빌드 · 실행이 안 됨) - android/build.gradle: compileSdk 34 → 36 (ffmpeg-kit min-gpl 이 35 이상을 요구) + +## 음성인식 취소 (JJ_MKVMaker) +- android/src/whisper/main.cpp: 요청 JSON 의 `abort_flag` (Dart 가 가진 int32 의 주소) 를 `whisper_full_params.abort_callback` 에 연결. 값이 0 이 아니면 whisper 가 다음 계산 단계에서 멈춘다. + Android arm64 의 힙 주소는 맨 위 바이트에 태그가 있어 Dart int 로는 음수 → 부호 있는 정수도 받는다. +- lib/src/whisper.dart: `transcribe(abortFlagAddress:)` → 요청 JSON 에 `abort_flag` 추가. diff --git a/packages/whisper_ggml/android/src/whisper/main.cpp b/packages/whisper_ggml/android/src/whisper/main.cpp index daec633..2418884 100644 --- a/packages/whisper_ggml/android/src/whisper/main.cpp +++ b/packages/whisper_ggml/android/src/whisper/main.cpp @@ -74,6 +74,9 @@ struct whisper_params // Address of a Dart NativeCallable; 0 = none. uint64_t progress_cb_addr = 0; + // JJ: address of an int32 cancel flag owned by Dart; non-zero = abort. 0 = none. + uint64_t abort_flag_addr = 0; + // Park the loaded model in g_model_cache after this request instead of // freeing it, so the next request with the same model file skips the // multi-second load (issue #26). Off = load-per-request, as always. @@ -166,6 +169,14 @@ json transcribe(json jsonBody) noexcept { params.progress_cb_addr = jsonBody["progress_callback"].get(); } + // JJ: Android arm64 heap pointers carry a tag in the top byte (0xb4…), so Dart's int + // for the address is negative — accept signed numbers too and reinterpret the bits. + if (jsonBody.contains("abort_flag") && jsonBody["abort_flag"].is_number_integer()) + { + params.abort_flag_addr = jsonBody["abort_flag"].is_number_unsigned() + ? jsonBody["abort_flag"].get() + : (uint64_t)jsonBody["abort_flag"].get(); + } if (jsonBody.contains("keep_model_loaded") && jsonBody["keep_model_loaded"].is_boolean()) { params.keep_model_loaded = jsonBody["keep_model_loaded"].get(); @@ -351,6 +362,14 @@ json transcribe(json jsonBody) noexcept wparams.progress_callback_user_data = (void *)(uintptr_t)params.progress_cb_addr; } + if (params.abort_flag_addr) { + // JJ: the user cancelled — whisper checks this between encoder/decoder steps + wparams.abort_callback = [](void * data) { + return *(volatile int32_t *)data != 0; + }; + wparams.abort_callback_user_data = (void *)(uintptr_t)params.abort_flag_addr; + } + if (whisper_full(ctx, wparams, pcmf32.data(), pcmf32.size()) != 0) { jsonResult["@type"] = "error"; diff --git a/packages/whisper_ggml/lib/src/whisper.dart b/packages/whisper_ggml/lib/src/whisper.dart index 1fd3948..a21d6f4 100644 --- a/packages/whisper_ggml/lib/src/whisper.dart +++ b/packages/whisper_ggml/lib/src/whisper.dart @@ -54,10 +54,17 @@ class Whisper { Future> _request({ required WhisperRequestDto whisperRequest, + Map? extra, }) async { return Isolate.run(() async { final Pointer data = - whisperRequest.toRequestString().toNativeUtf8(); + (extra == null + ? whisperRequest.toRequestString() + : json.encode({ + ...json.decode(whisperRequest.toRequestString()) as Map, + ...extra, + })) + .toNativeUtf8(); final DynamicLibrary lib = _openLib(); final Pointer res = lib.lookupFunction('request').call(data); @@ -80,6 +87,8 @@ class Whisper { required TranscribeRequest transcribeRequest, required String modelPath, void Function(int percent)? onProgress, + // JJ: address of an int32 flag; set it to non-zero to stop recognition early + int? abortFlagAddress, }) async { // A listener callable may be invoked from whisper's worker thread; // it delivers to this isolate. Kept open until the request finishes. @@ -105,6 +114,7 @@ class Whisper { modelPath, progressCallbackAddress: progressCallable?.nativeFunction.address, ), + extra: abortFlagAddress == null ? null : {'abort_flag': abortFlagAddress}, ); if (result['text'] == null) { diff --git a/test/ai_test.dart b/test/ai_test.dart index 041fc68..6592b1b 100644 --- a/test/ai_test.dart +++ b/test/ai_test.dart @@ -16,6 +16,8 @@ import 'package:jj_mkvmaker/services/platform_services.dart'; import 'package:path/path.dart' as p; class _FakeRecognizer implements SpeechRecognizer { + @override + void cancel() {} String? lastLanguage; @override Future> transcribe(String wavPath, diff --git a/test/batch_test.dart b/test/batch_test.dart index d2fb0d0..f0765d1 100644 --- a/test/batch_test.dart +++ b/test/batch_test.dart @@ -24,6 +24,8 @@ import 'package:jj_mkvmaker/ui/settings_page.dart'; import 'package:path/path.dart' as p; class _Recognizer implements SpeechRecognizer { + @override + void cancel() {} final heard = []; @override Future> transcribe(String wavPath,