From d5eeb12ce7dfe35bfce6d1c18d4870dc6aefd68b Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Sun, 7 May 2023 17:53:19 +0100 Subject: [PATCH] Android: Fix voice typing --- ...ct-native-vosk-npm-0.1.12-76b1caaae8.patch | 127 ++++++++++++++++++ package.json | 3 +- .../app-mobile/components/screens/Note.tsx | 66 +++++++++ packages/app-mobile/ios/Podfile.lock | 37 +---- packages/app-mobile/package.json | 2 +- .../app-mobile/services/voiceTyping/vosk.ts | 2 +- packages/react-native-vosk/README.md | 2 + yarn.lock | 24 +++- 8 files changed, 226 insertions(+), 37 deletions(-) create mode 100644 .yarn/patches/react-native-vosk-npm-0.1.12-76b1caaae8.patch diff --git a/.yarn/patches/react-native-vosk-npm-0.1.12-76b1caaae8.patch b/.yarn/patches/react-native-vosk-npm-0.1.12-76b1caaae8.patch new file mode 100644 index 0000000000..eda87c36ed --- /dev/null +++ b/.yarn/patches/react-native-vosk-npm-0.1.12-76b1caaae8.patch @@ -0,0 +1,127 @@ +diff --git a/android/build.gradle b/android/build.gradle +index 6afcbbf0cc8ca2d69dd78077d61e59a90b2136bb..9f8d72b4ec5b2b3d290975d6a255917c95300854 100644 +--- a/android/build.gradle ++++ b/android/build.gradle +@@ -67,19 +67,19 @@ repositories { + } + + // Generate UUIDs for each models contained in android/src/main/assets/ +-tasks.register('genUUID') { +- doLast { +- fileTree(dir: "$rootDir/app/src/main/assets", exclude: ['*/*']).visit { fileDetails -> +- if (fileDetails.directory) { +- def odir = file("$rootDir/app/src/main/assets/$fileDetails.relativePath") +- def ofile = file("$odir/uuid") +- mkdir odir +- ofile.text = UUID.randomUUID().toString() +- } +- } +- } +-} +-preBuild.dependsOn genUUID ++// tasks.register('genUUID') { ++// doLast { ++// fileTree(dir: "$rootDir/app/src/main/assets", exclude: ['*/*']).visit { fileDetails -> ++// if (fileDetails.directory) { ++// def odir = file("$rootDir/app/src/main/assets/$fileDetails.relativePath") ++// def ofile = file("$odir/uuid") ++// mkdir odir ++// ofile.text = UUID.randomUUID().toString() ++// } ++// } ++// } ++// } ++// preBuild.dependsOn genUUID + + def kotlin_version = getExtOrDefault('kotlinVersion') + +diff --git a/android/src/main/java/com/reactnativevosk/VoskModule.kt b/android/src/main/java/com/reactnativevosk/VoskModule.kt +index 0e2b6595b1b2cf1ee01c6c64239c4b0ea37fce19..f3da440bc2863a59db6d2d1691c54d8d4870cb3f 100644 +--- a/android/src/main/java/com/reactnativevosk/VoskModule.kt ++++ b/android/src/main/java/com/reactnativevosk/VoskModule.kt +@@ -19,13 +19,25 @@ class VoskModule(reactContext: ReactApplicationContext) : ReactContextBaseJavaMo + return "Vosk" + } + ++ @ReactMethod ++ fun addListener(type: String?) { ++ // Keep: Required for RN built in Event Emitter Calls. ++ } ++ ++ @ReactMethod ++ fun removeListeners(type: Int?) { ++ // Keep: Required for RN built in Event Emitter Calls. ++ } ++ + override fun onResult(hypothesis: String) { + // Get text data from string object + val text = getHypothesisText(hypothesis) + + // Stop recording if data found + if (text != null && text.isNotEmpty()) { +- cleanRecognizer(); ++ // Don't auto-stop the recogniser - we want to do that when the user ++ // presses on "stop" only. ++ // cleanRecognizer(); + sendEvent("onResult", text) + } + } +@@ -153,6 +165,25 @@ class VoskModule(reactContext: ReactApplicationContext) : ReactContextBaseJavaMo + cleanRecognizer(); + } + ++ @ReactMethod ++ fun stopOnly() { ++ if (speechService != null) { ++ speechService!!.stop() ++ } ++ } ++ ++ @ReactMethod ++ fun cleanup() { ++ if (speechService != null) { ++ speechService!!.shutdown(); ++ speechService = null ++ } ++ if (recognizer != null) { ++ recognizer!!.close(); ++ recognizer = null; ++ } ++ } ++ + @ReactMethod + fun unload() { + cleanRecognizer(); +diff --git a/lib/typescript/index.d.ts b/lib/typescript/index.d.ts +index 441e41cc402cca3a60b34978ef4fea976076259c..a173acebb4b314402550442ad471e0f7c706e3c4 100644 +--- a/lib/typescript/index.d.ts ++++ b/lib/typescript/index.d.ts +@@ -10,6 +10,8 @@ export default class Vosk { + currentRegisteredEvents: EmitterSubscription[]; + start: (grammar?: string[] | null) => Promise; + stop: () => void; ++ stopOnly: () => void; ++ cleanup: () => void; + unload: () => void; + onResult: (onResult: (e: VoskEvent) => void) => EventSubscription; + onFinalResult: (onFinalResult: (e: VoskEvent) => void) => EventSubscription; +diff --git a/src/index.tsx b/src/index.tsx +index d9f90c921d89b1b4d85e145443ed3376546a368a..29e4068dbd7500828a73145bd25497a52c9bf638 100644 +--- a/src/index.tsx ++++ b/src/index.tsx +@@ -69,6 +69,15 @@ export default class Vosk { + VoskModule.stop(); + }; + ++ stopOnly = () => { ++ VoskModule.stopOnly(); ++ }; ++ ++ cleanup = () => { ++ this.cleanListeners(); ++ VoskModule.cleanup(); ++ }; ++ + unload = () => { + this.cleanListeners(); + VoskModule.unload(); diff --git a/package.json b/package.json index a3b10492a1..d4c231b56e 100644 --- a/package.json +++ b/package.json @@ -96,6 +96,7 @@ "packageManager": "yarn@3.3.1", "resolutions": { "react-native-camera@4.2.1": "patch:react-native-camera@npm%3A4.2.1#./.yarn/patches/react-native-camera-npm-4.2.1-24b2600a7e.patch", - "rn-fetch-blob@0.12.0": "patch:rn-fetch-blob@npm%3A0.12.0#./.yarn/patches/rn-fetch-blob-npm-0.12.0-cf02e3c544.patch" + "rn-fetch-blob@0.12.0": "patch:rn-fetch-blob@npm%3A0.12.0#./.yarn/patches/rn-fetch-blob-npm-0.12.0-cf02e3c544.patch", + "react-native-vosk@0.1.12": "patch:react-native-vosk@npm%3A0.1.12#./.yarn/patches/react-native-vosk-npm-0.1.12-76b1caaae8.patch" } } diff --git a/packages/app-mobile/components/screens/Note.tsx b/packages/app-mobile/components/screens/Note.tsx index 525ebed570..76275163ba 100644 --- a/packages/app-mobile/components/screens/Note.tsx +++ b/packages/app-mobile/components/screens/Note.tsx @@ -48,6 +48,8 @@ import VoiceTypingDialog from '../voiceTyping/VoiceTypingDialog'; import { voskEnabled } from '../../services/voiceTyping/vosk'; const urlUtils = require('@joplin/lib/urlUtils'); +// import Vosk from 'react-native-vosk'; + const emptyArray: any[] = []; const logger = Logger.create('screens/Note'); @@ -878,6 +880,69 @@ class NoteScreenComponent extends BaseScreenComponent { if (buttonId === 'attachPhoto') void this.attachPhoto_onPress(); } + // private vosk_:Vosk; + + // private async getVosk() { + // if (this.vosk_) return this.vosk_; + // this.vosk_ = new Vosk(); + // await this.vosk_.loadModel('model-fr-fr'); + // return this.vosk_; + // } + + // private async voiceRecording_onPress() { + // logger.info('Vosk: Getting instance...'); + + // const vosk = await this.getVosk(); + + // this.voskResult_ = []; + + // const eventHandlers: any[] = []; + + // eventHandlers.push(vosk.onResult(e => { + // logger.info('Vosk: result', e.data); + // this.voskResult_.push(e.data); + // })); + + // eventHandlers.push(vosk.onError(e => { + // logger.warn('Vosk: error', e.data); + // })); + + // eventHandlers.push(vosk.onTimeout(e => { + // logger.warn('Vosk: timeout', e.data); + // })); + + // eventHandlers.push(vosk.onFinalResult(e => { + // logger.info('Vosk: final result', e.data); + // })); + + // logger.info('Vosk: Starting recording...'); + + // void vosk.start(); + + // const buttonId = await dialogs.pop(this, 'Voice recording in progress...', [ + // { text: 'Stop recording', id: 'stop' }, + // { text: _('Cancel'), id: 'cancel' }, + // ]); + + // logger.info('Vosk: Stopping recording...'); + // vosk.stop(); + + // for (const eventHandler of eventHandlers) { + // eventHandler.remove(); + // } + + // logger.info('Vosk: Recording stopped:', this.voskResult_); + + // if (buttonId === 'cancel') return; + + // const newNote: NoteEntity = { ...this.state.note }; + // newNote.body = `${newNote.body} ${this.voskResult_.join(' ')}`; + // this.setState({ note: newNote }); + // this.scheduleSave(); + // } + + + public menuOptions() { const note = this.state.note; const isTodo = note && !!note.is_todo; @@ -927,6 +992,7 @@ class NoteScreenComponent extends BaseScreenComponent { output.push({ title: _('Voice typing...'), onPress: () => { + // this.voiceRecording_onPress(); this.setState({ voiceTypingDialogShown: true }); }, }); diff --git a/packages/app-mobile/ios/Podfile.lock b/packages/app-mobile/ios/Podfile.lock index 5b9cf3775b..92973307b7 100644 --- a/packages/app-mobile/ios/Podfile.lock +++ b/packages/app-mobile/ios/Podfile.lock @@ -73,7 +73,6 @@ PODS: - FlipperKit/FlipperKitNetworkPlugin - fmt (6.2.1) - glog (0.3.5) - - hermes-engine (0.70.6) - JoplinCommonShareExtension (1.0.0) - JoplinRNShareExtension (1.0.0): - JoplinCommonShareExtension @@ -91,12 +90,6 @@ PODS: - DoubleConversion - fmt (~> 6.2.1) - glog - - RCT-Folly/Futures (2021.07.22.00): - - boost - - DoubleConversion - - fmt (~> 6.2.1) - - glog - - libevent - RCTRequired (0.70.6) - RCTTypeSafety (0.70.6): - FBLazyVector (= 0.70.6) @@ -274,17 +267,6 @@ PODS: - React-logger (= 0.70.6) - React-perflogger (= 0.70.6) - React-runtimeexecutor (= 0.70.6) - - React-hermes (0.70.6): - - DoubleConversion - - glog - - hermes-engine - - RCT-Folly (= 2021.07.22.00) - - RCT-Folly/Futures (= 2021.07.22.00) - - React-cxxreact (= 0.70.6) - - React-jsi (= 0.70.6) - - React-jsiexecutor (= 0.70.6) - - React-jsinspector (= 0.70.6) - - React-perflogger (= 0.70.6) - React-jsi (0.70.6): - boost (= 1.76.0) - DoubleConversion @@ -346,7 +328,7 @@ PODS: - React-Core - react-native-version-info (1.1.1): - React-Core - - react-native-vosk (0.1.13): + - react-native-vosk (0.1.12): - React-Core - react-native-webview (11.26.1): - React-Core @@ -470,10 +452,8 @@ DEPENDENCIES: - FlipperKit/FlipperKitUserDefaultsPlugin (= 0.125.0) - FlipperKit/SKIOSNetworkPlugin (= 0.125.0) - glog (from `../node_modules/react-native/third-party-podspecs/glog.podspec`) - - hermes-engine (from `../node_modules/react-native/sdks/hermes/hermes-engine.podspec`) - JoplinCommonShareExtension (from `ShareExtension`) - JoplinRNShareExtension (from `ShareExtension`) - - libevent (~> 2.1.12) - OpenSSL-Universal (= 1.1.1100) - RCT-Folly (from `../node_modules/react-native/third-party-podspecs/RCT-Folly.podspec`) - RCTRequired (from `../node_modules/react-native/Libraries/RCTRequired`) @@ -487,7 +467,6 @@ DEPENDENCIES: - React-Core/RCTWebSocket (from `../node_modules/react-native/`) - React-CoreModules (from `../node_modules/react-native/React/CoreModules`) - React-cxxreact (from `../node_modules/react-native/ReactCommon/cxxreact`) - - React-hermes (from `../node_modules/react-native/ReactCommon/hermes`) - React-jsi (from `../node_modules/react-native/ReactCommon/jsi`) - React-jsiexecutor (from `../node_modules/react-native/ReactCommon/jsiexecutor`) - React-jsinspector (from `../node_modules/react-native/ReactCommon/jsinspector`) @@ -507,7 +486,7 @@ DEPENDENCIES: - "react-native-slider (from `../node_modules/@react-native-community/slider`)" - react-native-sqlite-storage (from `../node_modules/react-native-sqlite-storage`) - react-native-version-info (from `../node_modules/react-native-version-info`) - - "react-native-vosk (from `../node_modules/@joplin/react-native-vosk`)" + - react-native-vosk (from `../node_modules/react-native-vosk`) - react-native-webview (from `../node_modules/react-native-webview`) - React-perflogger (from `../node_modules/react-native/ReactCommon/reactperflogger`) - React-RCTActionSheet (from `../node_modules/react-native/Libraries/ActionSheetIOS`) @@ -563,8 +542,6 @@ EXTERNAL SOURCES: :path: "../node_modules/react-native/React/FBReactNativeSpec" glog: :podspec: "../node_modules/react-native/third-party-podspecs/glog.podspec" - hermes-engine: - :podspec: "../node_modules/react-native/sdks/hermes/hermes-engine.podspec" JoplinCommonShareExtension: :path: ShareExtension JoplinRNShareExtension: @@ -589,8 +566,6 @@ EXTERNAL SOURCES: :path: "../node_modules/react-native/React/CoreModules" React-cxxreact: :path: "../node_modules/react-native/ReactCommon/cxxreact" - React-hermes: - :path: "../node_modules/react-native/ReactCommon/hermes" React-jsi: :path: "../node_modules/react-native/ReactCommon/jsi" React-jsiexecutor: @@ -630,7 +605,7 @@ EXTERNAL SOURCES: react-native-version-info: :path: "../node_modules/react-native-version-info" react-native-vosk: - :path: "../node_modules/@joplin/react-native-vosk" + :path: "../node_modules/react-native-vosk" react-native-webview: :path: "../node_modules/react-native-webview" React-perflogger: @@ -699,7 +674,6 @@ SPEC CHECKSUMS: FlipperKit: cbdee19bdd4e7f05472a66ce290f1b729ba3cb86 fmt: ff9d55029c625d3757ed641535fd4a75fedc7ce9 glog: 04b94705f318337d7ead9e6d17c019bd9b1f6b1b - hermes-engine: 2af7b7a59128f250adfd86f15aa1d5a2ecd39995 JoplinCommonShareExtension: a8b60b02704d85a7305627912c0240e94af78db7 JoplinRNShareExtension: 485f3e6dad83b7b77f1572eabc249f869ee55c02 libevent: 4049cae6c81cdb3654a443be001fb9bdceff7913 @@ -714,7 +688,6 @@ SPEC CHECKSUMS: React-Core: b587d0a624f9611b0e032505f3d6f25e8daa2bee React-CoreModules: c6ff48b985e7aa622e82ca51c2c353c7803eb04e React-cxxreact: ade3d9e63c599afdead3c35f8a8bd12b3da6730b - React-hermes: ed09ae33512bbb8d31b2411778f3af1a2eb681a1 React-jsi: 5a3952e0c6d57460ad9ee2c905025b4c28f71087 React-jsiexecutor: b4a65947391c658450151275aa406f2b8263178f React-jsinspector: 60769e5a0a6d4b32294a2456077f59d0266f9a8b @@ -734,7 +707,7 @@ SPEC CHECKSUMS: react-native-slider: 33b8d190b59d4f67a541061bb91775d53d617d9d react-native-sqlite-storage: f6d515e1c446d1e6d026aa5352908a25d4de3261 react-native-version-info: a106f23009ac0db4ee00de39574eb546682579b9 - react-native-vosk: 2e94a03a6b9a45c512b22810ac5ac111d0925bcb + react-native-vosk: 33b8e82a46cc56f31bb4847a40efa2d160270e2e react-native-webview: 9f111dfbcfc826084d6c507f569e5e03342ee1c1 React-perflogger: 8c79399b0500a30ee8152d0f9f11beae7fc36595 React-RCTActionSheet: 7316773acabb374642b926c19aef1c115df5c466 @@ -763,6 +736,6 @@ SPEC CHECKSUMS: Yoga: 99caf8d5ab45e9d637ee6e0174ec16fbbb01bcfc YogaKit: f782866e155069a2cca2517aafea43200b01fd5a -PODFILE CHECKSUM: 1f5ea1b29b693e847adf004360d019d064a024ca +PODFILE CHECKSUM: 0235ffbfa2e655de806a80d996148182dd493d8d COCOAPODS: 1.11.3 diff --git a/packages/app-mobile/package.json b/packages/app-mobile/package.json index 47d65b43ec..84368a2ad3 100644 --- a/packages/app-mobile/package.json +++ b/packages/app-mobile/package.json @@ -21,7 +21,6 @@ "@joplin/lib": "~2.11", "@joplin/react-native-alarm-notification": "~2.11", "@joplin/react-native-saf-x": "~2.11", - "@joplin/react-native-vosk": "~0.1", "@joplin/renderer": "~2.11", "@react-native-community/clipboard": "1.5.1", "@react-native-community/datetimepicker": "6.7.5", @@ -67,6 +66,7 @@ "react-native-url-polyfill": "1.3.0", "react-native-vector-icons": "9.2.0", "react-native-version-info": "1.1.1", + "react-native-vosk": "0.1.12", "react-native-webview": "11.26.1", "react-redux": "7.2.9", "redux": "4.2.1", diff --git a/packages/app-mobile/services/voiceTyping/vosk.ts b/packages/app-mobile/services/voiceTyping/vosk.ts index 3cd4d8f912..bb70b46050 100644 --- a/packages/app-mobile/services/voiceTyping/vosk.ts +++ b/packages/app-mobile/services/voiceTyping/vosk.ts @@ -1,5 +1,5 @@ import Logger from '@joplin/lib/Logger'; -import Vosk from '@joplin/react-native-vosk'; +import Vosk from 'react-native-vosk'; const logger = Logger.create('voiceTyping/vosk'); enum State { diff --git a/packages/react-native-vosk/README.md b/packages/react-native-vosk/README.md index b7d3a1401f..dd54588e49 100644 --- a/packages/react-native-vosk/README.md +++ b/packages/react-native-vosk/README.md @@ -2,6 +2,8 @@ * * * +**NOTE:** For some reason this module doesn't work (events are not being fired), so for now we use the actual `react-native-vosk` module, but with a patch + **Joplin fork** of `react-native-vosk@0.1.12` with the following changes: - The `onResult()` event doesn't automatically stop the recorder - because we need it to keep running so that it captures the whole text. The original package was designed to record one keyword, but we need whole sentences. diff --git a/yarn.lock b/yarn.lock index 6cb2328e0b..b2c3f89343 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6339,7 +6339,6 @@ __metadata: "@joplin/lib": ~2.11 "@joplin/react-native-alarm-notification": ~2.11 "@joplin/react-native-saf-x": ~2.11 - "@joplin/react-native-vosk": ~0.1 "@joplin/renderer": ~2.11 "@joplin/tools": ~2.11 "@lezer/highlight": 1.1.4 @@ -6402,6 +6401,7 @@ __metadata: react-native-url-polyfill: 1.3.0 react-native-vector-icons: 9.2.0 react-native-version-info: 1.1.1 + react-native-vosk: 0.1.12 react-native-webview: 11.26.1 react-redux: 7.2.9 redux: 4.2.1 @@ -6642,7 +6642,7 @@ __metadata: languageName: unknown linkType: soft -"@joplin/react-native-vosk@workspace:packages/react-native-vosk, @joplin/react-native-vosk@~0.1": +"@joplin/react-native-vosk@workspace:packages/react-native-vosk": version: 0.0.0-use.local resolution: "@joplin/react-native-vosk@workspace:packages/react-native-vosk" dependencies: @@ -31745,6 +31745,26 @@ __metadata: languageName: node linkType: hard +"react-native-vosk@npm:0.1.12": + version: 0.1.12 + resolution: "react-native-vosk@npm:0.1.12" + peerDependencies: + react: "*" + react-native: "*" + checksum: 49dd234d0822d7f3deb9563a903260a8478bb78eb20367b50284df40e1e64e23dc52d632b329176883c048b8224182eee000fd7dbd3c42401a9a03bd0ce1ae10 + languageName: node + linkType: hard + +"react-native-vosk@patch:react-native-vosk@npm%3A0.1.12#./.yarn/patches/react-native-vosk-npm-0.1.12-76b1caaae8.patch::locator=root%40workspace%3A.": + version: 0.1.12 + resolution: "react-native-vosk@patch:react-native-vosk@npm%3A0.1.12#./.yarn/patches/react-native-vosk-npm-0.1.12-76b1caaae8.patch::version=0.1.12&hash=8a1ee1&locator=root%40workspace%3A." + peerDependencies: + react: "*" + react-native: "*" + checksum: 42ca9a0a4f60706b718c95207fcc27ec43571c0a5e83b358422fd35e45d006762277c30556edb1b470ade40198076d69260c4233fd07e30cd33c0f03024f5d11 + languageName: node + linkType: hard + "react-native-webview@npm:11.26.1": version: 11.26.1 resolution: "react-native-webview@npm:11.26.1"