From 6220267abb962fb0e75f071113dd18a301be6681 Mon Sep 17 00:00:00 2001 From: Henry Heino <46334387+personalizedrefrigerator@users.noreply.github.com> Date: Mon, 6 Jan 2025 09:32:19 -0800 Subject: [PATCH] Mobile: Upgrade js-draw to 1.26.0 (#11589) --- .eslintignore | 1 + .gitignore | 1 + .../components/ExtendedWebView/index.web.tsx | 2 +- .../NoteEditor/ImageEditor/ImageEditor.tsx | 133 +++++------------- .../js-draw/createJsDrawEditor.test.ts | 8 +- .../ImageEditor/js-draw/createJsDrawEditor.ts | 68 ++++++--- .../ImageEditor/js-draw/startAutosaveLoop.ts | 2 +- .../NoteEditor/ImageEditor/js-draw/types.ts | 17 ++- .../ImageEditor/utils/useEditorMessenger.ts | 63 +++++++++ packages/app-mobile/package.json | 4 +- yarn.lock | 30 ++-- 11 files changed, 182 insertions(+), 147 deletions(-) create mode 100644 packages/app-mobile/components/NoteEditor/ImageEditor/utils/useEditorMessenger.ts diff --git a/.eslintignore b/.eslintignore index 1d95a55980..fac3684314 100644 --- a/.eslintignore +++ b/.eslintignore @@ -644,6 +644,7 @@ packages/app-mobile/components/NoteEditor/ImageEditor/js-draw/startAutosaveLoop. packages/app-mobile/components/NoteEditor/ImageEditor/js-draw/types.js packages/app-mobile/components/NoteEditor/ImageEditor/js-draw/watchEditorForTemplateChanges.js packages/app-mobile/components/NoteEditor/ImageEditor/promptRestoreAutosave.js +packages/app-mobile/components/NoteEditor/ImageEditor/utils/useEditorMessenger.js packages/app-mobile/components/NoteEditor/NoteEditor.test.js packages/app-mobile/components/NoteEditor/NoteEditor.js packages/app-mobile/components/NoteEditor/SearchPanel.js diff --git a/.gitignore b/.gitignore index ac6b0fcfed..5e124cc591 100644 --- a/.gitignore +++ b/.gitignore @@ -619,6 +619,7 @@ packages/app-mobile/components/NoteEditor/ImageEditor/js-draw/startAutosaveLoop. packages/app-mobile/components/NoteEditor/ImageEditor/js-draw/types.js packages/app-mobile/components/NoteEditor/ImageEditor/js-draw/watchEditorForTemplateChanges.js packages/app-mobile/components/NoteEditor/ImageEditor/promptRestoreAutosave.js +packages/app-mobile/components/NoteEditor/ImageEditor/utils/useEditorMessenger.js packages/app-mobile/components/NoteEditor/NoteEditor.test.js packages/app-mobile/components/NoteEditor/NoteEditor.js packages/app-mobile/components/NoteEditor/SearchPanel.js diff --git a/packages/app-mobile/components/ExtendedWebView/index.web.tsx b/packages/app-mobile/components/ExtendedWebView/index.web.tsx index 39ae58a6f0..8383a2828e 100644 --- a/packages/app-mobile/components/ExtendedWebView/index.web.tsx +++ b/packages/app-mobile/components/ExtendedWebView/index.web.tsx @@ -114,7 +114,7 @@ const ExtendedWebView = (props: Props, ref: Ref) => { // allow-popups-to-escape-sandbox: Allows PDF previews to work on target="_blank" links. // allow-popups: Allows links to open in a new tab. permissions: 'allow-scripts allow-modals allow-popups allow-popups-to-escape-sandbox', - allow: 'clipboard-write=(self) fullscreen=(self) autoplay=(self) local-fonts=* encrypted-media=*', + allow: 'clipboard-write; clipboard-read; fullscreen \'self\'; autoplay \'self\'; local-fonts \'self\'; encrypted-media \'self\'', }); if (containerRef.current) { diff --git a/packages/app-mobile/components/NoteEditor/ImageEditor/ImageEditor.tsx b/packages/app-mobile/components/NoteEditor/ImageEditor/ImageEditor.tsx index dccd6eb57c..82356b2fb8 100644 --- a/packages/app-mobile/components/NoteEditor/ImageEditor/ImageEditor.tsx +++ b/packages/app-mobile/components/NoteEditor/ImageEditor/ImageEditor.tsx @@ -8,12 +8,11 @@ import { Theme } from '@joplin/lib/themes/type'; import { MutableRefObject, useCallback, useContext, useEffect, useMemo, useRef, useState } from 'react'; import { BackHandler, Platform } from 'react-native'; import ExtendedWebView from '../../ExtendedWebView'; -import { WebViewControl } from '../../ExtendedWebView/types'; -import { clearAutosave, writeAutosave } from './autosave'; +import { OnMessageEvent, WebViewControl } from '../../ExtendedWebView/types'; +import { clearAutosave } from './autosave'; import { LocalizedStrings } from './js-draw/types'; -import VersionInfo from 'react-native-version-info'; import { DialogContext } from '../../DialogManager'; -import { OnMessageEvent } from '../../ExtendedWebView/types'; +import useEditorMessenger from './utils/useEditorMessenger'; const logger = Logger.create('ImageEditor'); @@ -172,7 +171,7 @@ const ImageEditor = (props: Props) => { const appInfo = useMemo(() => { return { name: 'Joplin', - description: `v${VersionInfo.appVersion}`, + description: `v${shim.appVersion()}`, }; }, []); @@ -189,79 +188,23 @@ const ImageEditor = (props: Props) => { ); }; - const setImageHasChanges = (hasChanges) => { - window.ReactNativeWebView.postMessage( - JSON.stringify({ - action: 'set-image-has-changes', - data: hasChanges, - }), - ); - }; - - window.updateEditorTemplate = (templateData) => { - window.ReactNativeWebView.postMessage( - JSON.stringify({ - action: 'set-image-template-data', - data: templateData, - }), - ); - }; - - const notifyReadyToLoadSVG = () => { - window.ReactNativeWebView.postMessage( - JSON.stringify({ - action: 'ready-to-load-data', - }) - ); - }; - - const saveDrawing = async (drawing, isAutosave) => { - window.ReactNativeWebView.postMessage( - JSON.stringify({ - action: isAutosave ? 'autosave' : 'save', - data: drawing.outerHTML, - }), - ); - }; - - const closeEditor = (promptIfUnsaved) => { - window.ReactNativeWebView.postMessage(JSON.stringify({ - action: 'close', - promptIfUnsaved, - })); - }; - - const saveThenClose = (drawing) => { - window.ReactNativeWebView.postMessage( - JSON.stringify({ - action: 'save-and-close', - data: drawing.outerHTML, - }), - ); - }; - try { if (window.editorControl === undefined) { ${shim.injectedJs('svgEditorBundle')} window.editorControl = svgEditorBundle.createJsDrawEditor( - { - saveDrawing, - closeEditor, - saveThenClose, - updateEditorTemplate, - setImageHasChanges, - }, + svgEditorBundle.createMessenger().remoteApi, ${JSON.stringify(Setting.value('imageeditor.jsdrawToolbar'))}, ${JSON.stringify(Setting.value('locale'))}, ${JSON.stringify(localizedStrings)}, - ${JSON.stringify({ appInfo })}, + ${JSON.stringify({ + appInfo, + ...(shim.mobilePlatform() === 'web' ? { + // Use the browser-default clipboard API on web. + clipboardApi: null, + } : {}), + })}, ); - - // Start loading the SVG file (if present) after loading the editor. - // This shows the user that progress is being made (loading large SVGs - // from disk into memory can take several seconds). - notifyReadyToLoadSVG(); } } catch(e) { window.ReactNativeWebView.postMessage( @@ -273,7 +216,7 @@ const ImageEditor = (props: Props) => { useEffect(() => { webviewRef.current?.injectJS(` - document.querySelector('#main-style').innerText = ${JSON.stringify(css)}; + document.querySelector('#main-style').textContent = ${JSON.stringify(css)}; if (window.editorControl) { window.editorControl.onThemeUpdate(); @@ -308,48 +251,36 @@ const ImageEditor = (props: Props) => { })();`); }, [webviewRef, props.resourceFilename]); - const onMessage = useCallback(async (event: OnMessageEvent) => { - const data = event.nativeEvent.data; - if (data.startsWith('error:')) { - logger.error('ImageEditor:', data); - return; - } - - const json = JSON.parse(data); - if (json.action === 'save') { - await clearAutosave(); - await props.onSave(json.data); - } else if (json.action === 'autosave') { - await writeAutosave(json.data); - } else if (json.action === 'save-toolbar') { - Setting.setValue('imageeditor.jsdrawToolbar', json.data); - } else if (json.action === 'close') { - onRequestCloseEditor(json.promptIfUnsaved); - } else if (json.action === 'save-and-close') { - await props.onSave(json.data); - onRequestCloseEditor(json.promptIfUnsaved); - } else if (json.action === 'ready-to-load-data') { - void onReadyToLoadData(); - } else if (json.action === 'set-image-has-changes') { - setImageChanged(json.data); - } else if (json.action === 'set-image-template-data') { - Setting.setValue('imageeditor.imageTemplate', json.data); - } else { - logger.error('Unknown action,', json.action); - } - }, [props.onSave, onRequestCloseEditor, onReadyToLoadData]); - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied const onError = useCallback((event: any) => { logger.error('ImageEditor: WebView error: ', event); }, []); + const messenger = useEditorMessenger({ + webviewRef, + setImageChanged, + onReadyToLoadData, + onSave: props.onSave, + onRequestCloseEditor, + }); + + const onMessage = useCallback((event: OnMessageEvent) => { + const data = event.nativeEvent.data; + if (typeof data === 'string' && data.startsWith('error:')) { + logger.error(data); + return; + } + + messenger.onWebViewMessage(event); + }, [messenger]); + return ( ) => const locale = 'en'; const allCallbacks: ImageEditorCallbacks = { - saveDrawing: () => {}, + save: () => {}, saveThenClose: ()=> {}, closeEditor: ()=> {}, setImageHasChanges: ()=> {}, updateEditorTemplate: ()=> {}, + updateToolbarState: ()=> {}, + onLoadedEditor: ()=> {}, + writeClipboardText: async ()=>{}, + readClipboardText: async ()=> '', ...callbacks, }; @@ -51,7 +55,7 @@ describe('createJsDrawEditor', () => { jest.useFakeTimers(); const editorControl = createEditorWithCallbacks({ - saveDrawing: (_drawing: SVGElement, isAutosave: boolean) => { + save: (_drawing: string, isAutosave: boolean) => { if (isAutosave) { calledAutosaveCount ++; } diff --git a/packages/app-mobile/components/NoteEditor/ImageEditor/js-draw/createJsDrawEditor.ts b/packages/app-mobile/components/NoteEditor/ImageEditor/js-draw/createJsDrawEditor.ts index f82b2b2c0f..29d89a1b6d 100644 --- a/packages/app-mobile/components/NoteEditor/ImageEditor/js-draw/createJsDrawEditor.ts +++ b/packages/app-mobile/components/NoteEditor/ImageEditor/js-draw/createJsDrawEditor.ts @@ -4,13 +4,9 @@ import { MaterialIconProvider } from '@js-draw/material-icons'; import 'js-draw/bundledStyles'; import applyTemplateToEditor from './applyTemplateToEditor'; import watchEditorForTemplateChanges from './watchEditorForTemplateChanges'; -import { ImageEditorCallbacks, LocalizedStrings } from './types'; +import { ImageEditorCallbacks, ImageEditorControl, LocalizedStrings } from './types'; import startAutosaveLoop from './startAutosaveLoop'; - -declare namespace ReactNativeWebView { - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied - const postMessage: (data: any)=> void; -} +import WebViewToRNMessenger from '../../../../utils/ipc/WebViewToRNMessenger'; const restoreToolbarState = (toolbar: AbstractToolbar, state: string) => { if (state) { @@ -23,19 +19,13 @@ const restoreToolbarState = (toolbar: AbstractToolbar, state: string) => { } }; -const listenToolbarState = (editor: Editor, toolbar: AbstractToolbar) => { - editor.notifier.on(EditorEventType.ToolUpdated, () => { - const state = toolbar.serializeState(); - ReactNativeWebView.postMessage( - JSON.stringify({ - action: 'save-toolbar', - data: state, - }), - ); - }); +export const createMessenger = () => { + const messenger = new WebViewToRNMessenger( + 'image-editor', {}, + ); + return messenger; }; - export const createJsDrawEditor = ( callbacks: ImageEditorCallbacks, initialToolbarState: string, @@ -54,6 +44,38 @@ export const createJsDrawEditor = ( ...defaultLocalizations, }, iconProvider: new MaterialIconProvider(), + clipboardApi: { + read: async () => { + const result = new Map(); + + const clipboardText = await callbacks.readClipboardText(); + if (clipboardText) { + result.set('text/plain', clipboardText); + } + + return result; + }, + write: async (data) => { + const getTextForMime = async (mime: string) => { + const text = data.get(mime); + if (typeof text === 'string') { + return text; + } + if (text) { + return await (await text).text(); + } + return null; + }; + + const svgData = await getTextForMime('image/svg+xml'); + if (svgData) { + return callbacks.writeClipboardText(svgData); + } + + const textData = await getTextForMime('text/plain'); + return callbacks.writeClipboardText(textData); + }, + }, ...editorSettings, }); @@ -95,11 +117,11 @@ export const createJsDrawEditor = ( return editor.toSVG({ // Grow small images to this minimum size minDimension: 50, - }); + }).outerHTML; }; const saveNow = () => { - callbacks.saveDrawing(getEditorSVG(), false); + callbacks.save(getEditorSVG(), false); // The image is now up-to-date with the resource setImageHasChanges(false); @@ -109,7 +131,9 @@ export const createJsDrawEditor = ( // Load and save toolbar-related state (e.g. pen sizes/colors). restoreToolbarState(toolbar, initialToolbarState); - listenToolbarState(editor, toolbar); + editor.notifier.on(EditorEventType.ToolUpdated, () => { + callbacks.updateToolbarState(toolbar.serializeState()); + }); setImageHasChanges(false); @@ -171,7 +195,7 @@ export const createJsDrawEditor = ( // We can now edit and save safely (without data loss). editor.setReadOnly(false); - void startAutosaveLoop(editor, callbacks.saveDrawing); + void startAutosaveLoop(editor, callbacks.save); watchEditorForTemplateChanges(editor, templateData, callbacks.updateEditorTemplate); }, onThemeUpdate: () => { @@ -187,6 +211,8 @@ export const createJsDrawEditor = ( editorControl.onThemeUpdate(); + callbacks.onLoadedEditor(); + return editorControl; }; diff --git a/packages/app-mobile/components/NoteEditor/ImageEditor/js-draw/startAutosaveLoop.ts b/packages/app-mobile/components/NoteEditor/ImageEditor/js-draw/startAutosaveLoop.ts index 9c774a8132..88d1084e1f 100644 --- a/packages/app-mobile/components/NoteEditor/ImageEditor/js-draw/startAutosaveLoop.ts +++ b/packages/app-mobile/components/NoteEditor/ImageEditor/js-draw/startAutosaveLoop.ts @@ -11,7 +11,7 @@ const startAutosaveLoop = async ( const createAutosave = async () => { const savedSVG = await editor.toSVGAsync(); - saveDrawing(savedSVG, true); + saveDrawing(savedSVG.outerHTML, true); }; while (true) { diff --git a/packages/app-mobile/components/NoteEditor/ImageEditor/js-draw/types.ts b/packages/app-mobile/components/NoteEditor/ImageEditor/js-draw/types.ts index b0db5d8819..f21d7ea2a4 100644 --- a/packages/app-mobile/components/NoteEditor/ImageEditor/js-draw/types.ts +++ b/packages/app-mobile/components/NoteEditor/ImageEditor/js-draw/types.ts @@ -1,16 +1,25 @@ -export type SaveDrawingCallback = (svgElement: SVGElement, isAutosave: boolean)=> void; +export type SaveDrawingCallback = (svgData: string, isAutosave: boolean)=> void; export type UpdateEditorTemplateCallback = (newTemplate: string)=> void; +export type UpdateToolbarCallback = (toolbarData: string)=> void; export interface ImageEditorCallbacks { - saveDrawing: SaveDrawingCallback; - updateEditorTemplate: UpdateEditorTemplateCallback; + onLoadedEditor: ()=> void; - saveThenClose: (svgData: SVGElement)=> void; + save: SaveDrawingCallback; + updateEditorTemplate: UpdateEditorTemplateCallback; + updateToolbarState: UpdateToolbarCallback; + + saveThenClose: (svgData: string)=> void; closeEditor: (promptIfUnsaved: boolean)=> void; setImageHasChanges: (hasChanges: boolean)=> void; + + writeClipboardText: (text: string)=> Promise; + readClipboardText: ()=> Promise; } +export interface ImageEditorControl {} + // Overrides translations in js-draw -- as of the time of this writing, // Joplin has many common strings localized better than js-draw. export interface LocalizedStrings { diff --git a/packages/app-mobile/components/NoteEditor/ImageEditor/utils/useEditorMessenger.ts b/packages/app-mobile/components/NoteEditor/ImageEditor/utils/useEditorMessenger.ts new file mode 100644 index 0000000000..f380b204d5 --- /dev/null +++ b/packages/app-mobile/components/NoteEditor/ImageEditor/utils/useEditorMessenger.ts @@ -0,0 +1,63 @@ +import { RefObject, useMemo } from 'react'; +import { WebViewControl } from '../../../ExtendedWebView/types'; +import { ImageEditorCallbacks, ImageEditorControl } from '../js-draw/types'; +import Setting from '@joplin/lib/models/Setting'; +import RNToWebViewMessenger from '../../../../utils/ipc/RNToWebViewMessenger'; +import { writeAutosave } from '../autosave'; +import Clipboard from '@react-native-clipboard/clipboard'; + +interface Props { + webviewRef: RefObject; + setImageChanged(changed: boolean): void; + + onReadyToLoadData(): void; + onSave(data: string): void; + onRequestCloseEditor(promptIfUnsaved: boolean): void; +} + +const useEditorMessenger = ({ + webviewRef, setImageChanged, onReadyToLoadData, onRequestCloseEditor, onSave, +}: Props) => { + return useMemo(() => { + const localApi: ImageEditorCallbacks = { + updateEditorTemplate: newTemplate => { + Setting.setValue('imageeditor.imageTemplate', newTemplate); + }, + updateToolbarState: newData => { + Setting.setValue('imageeditor.jsdrawToolbar', newData); + }, + setImageHasChanges: hasChanges => { + setImageChanged(hasChanges); + }, + onLoadedEditor: () => { + onReadyToLoadData(); + }, + saveThenClose: svgData => { + onSave(svgData); + onRequestCloseEditor(false); + }, + save: (svgData, isAutosave) => { + if (isAutosave) { + return writeAutosave(svgData); + } else { + return onSave(svgData); + } + }, + closeEditor: promptIfUnsaved => { + onRequestCloseEditor(promptIfUnsaved); + }, + writeClipboardText: async text => { + Clipboard.setString(text); + }, + readClipboardText: async () => { + return Clipboard.getString(); + }, + }; + const messenger = new RNToWebViewMessenger( + 'image-editor', webviewRef, localApi, + ); + return messenger; + }, [webviewRef, setImageChanged, onReadyToLoadData, onRequestCloseEditor, onSave]); +}; + +export default useEditorMessenger; diff --git a/packages/app-mobile/package.json b/packages/app-mobile/package.json index b00fa4df94..20967b77b9 100644 --- a/packages/app-mobile/package.json +++ b/packages/app-mobile/package.json @@ -91,7 +91,7 @@ "@babel/preset-env": "7.24.7", "@babel/runtime": "7.24.7", "@joplin/tools": "~3.2", - "@js-draw/material-icons": "1.20.3", + "@js-draw/material-icons": "1.26.0", "@react-native/babel-preset": "0.74.86", "@react-native/metro-config": "0.74.87", "@sqlite.org/sqlite-wasm": "3.46.0-build2", @@ -116,7 +116,7 @@ "jest": "29.7.0", "jest-environment-jsdom": "29.7.0", "jetifier": "2.0.0", - "js-draw": "1.20.3", + "js-draw": "1.26.0", "jsdom": "24.1.1", "nodemon": "3.1.7", "punycode": "2.3.1", diff --git a/yarn.lock b/yarn.lock index 484c558139..f114860e16 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8363,7 +8363,7 @@ __metadata: "@joplin/renderer": ~3.2 "@joplin/tools": ~3.2 "@joplin/utils": ~3.2 - "@js-draw/material-icons": 1.20.3 + "@js-draw/material-icons": 1.26.0 "@react-native-clipboard/clipboard": 1.14.2 "@react-native-community/datetimepicker": 8.2.0 "@react-native-community/geolocation": 3.3.0 @@ -8403,7 +8403,7 @@ __metadata: jest: 29.7.0 jest-environment-jsdom: 29.7.0 jetifier: 2.0.0 - js-draw: 1.20.3 + js-draw: 1.26.0 jsdom: 24.1.1 lodash: 4.17.21 md5: 2.3.0 @@ -9130,21 +9130,21 @@ __metadata: languageName: node linkType: hard -"@js-draw/material-icons@npm:1.20.3": - version: 1.20.3 - resolution: "@js-draw/material-icons@npm:1.20.3" +"@js-draw/material-icons@npm:1.26.0": + version: 1.26.0 + resolution: "@js-draw/material-icons@npm:1.26.0" peerDependencies: js-draw: ^1.0.1 - checksum: e935651aa63baee92c0b91731a5eb3b3dea1b2468e1666a4b75618b61e77ac960fe6d187c3c9a58d8c1353a9cc342646c2ad3c91b30793c67ccf4ab89e1598c0 + checksum: 05b8bcc190cca9e193cb1b5c2646cbc768e8f21d492c7baade0a405a8dd22118fca4aad2774c5a84dc0a18e89d8aafe0ed2806fa64aafefd46bbb6d96414bde3 languageName: node linkType: hard -"@js-draw/math@npm:^1.19.0": - version: 1.19.0 - resolution: "@js-draw/math@npm:1.19.0" +"@js-draw/math@npm:^1.26.0": + version: 1.26.0 + resolution: "@js-draw/math@npm:1.26.0" dependencies: bezier-js: 6.1.3 - checksum: a82989485e286b034bf8699b7995b8f7b3dc58e8c81c53efe87a6244d2b47382f0217d7a8c445dbed325de191111b1571a2017332bc52136a972c93a5a976532 + checksum: 086b1b3c0b1afbb0cb8939ccb37458c0114d9ad0fb2523b585bafae18fd3c99b75fd1fd57feb1be0ef586296ea5af17d50acb98134603c0316ebcf1e68ab53b2 languageName: node linkType: hard @@ -30876,13 +30876,13 @@ __metadata: languageName: node linkType: hard -"js-draw@npm:1.20.3": - version: 1.20.3 - resolution: "js-draw@npm:1.20.3" +"js-draw@npm:1.26.0": + version: 1.26.0 + resolution: "js-draw@npm:1.26.0" dependencies: - "@js-draw/math": ^1.19.0 + "@js-draw/math": ^1.26.0 "@melloware/coloris": 0.22.0 - checksum: 0674ebabb1f54355c738c8b7d6c82137a906014fc328caae41f0ba705d08deb4666a79dae027f49801cad377a5fb2a9a46640ca5e04d61635fad4e06ad86835b + checksum: 8e59290a62d465da215fcc7cce63f6fd38e4585f95ff2007a107a7cbaf1a6f66f731bdfbff5ff82f006f23f9256e65742102b5f3547144cfa8ee4f22b4a5bb02 languageName: node linkType: hard