Mobile: Resolves #9017: Support pasting images (#10751)

.eslintignore

@@ -713,6 +713,8 @@ packages/app-mobile/utils/fs-driver/testUtil/createFilesFromPathRecord.js
 packages/app-mobile/utils/fs-driver/testUtil/verifyDirectoryMatches.js
 packages/app-mobile/utils/getPackageInfo.js
 packages/app-mobile/utils/getVersionInfoText.js
+packages/app-mobile/utils/image/getImageDimensions.js
+packages/app-mobile/utils/image/resizeImage.js
 packages/app-mobile/utils/initializeCommandService.js
 packages/app-mobile/utils/ipc/RNToWebViewMessenger.js
 packages/app-mobile/utils/ipc/WebViewToRNMessenger.js
@@ -795,6 +797,7 @@ packages/editor/CodeMirror/utils/formatting/toggleRegionFormatGlobally.js
 packages/editor/CodeMirror/utils/formatting/toggleSelectedLinesStartWith.js
 packages/editor/CodeMirror/utils/formatting/types.js
 packages/editor/CodeMirror/utils/growSelectionToNode.js
+packages/editor/CodeMirror/utils/handlePasteEvent.js
 packages/editor/CodeMirror/utils/isInSyntaxNode.js
 packages/editor/CodeMirror/utils/setupVim.js
 packages/editor/SelectionFormatting.js

.gitignore

@@ -692,6 +692,8 @@ packages/app-mobile/utils/fs-driver/testUtil/createFilesFromPathRecord.js
 packages/app-mobile/utils/fs-driver/testUtil/verifyDirectoryMatches.js
 packages/app-mobile/utils/getPackageInfo.js
 packages/app-mobile/utils/getVersionInfoText.js
+packages/app-mobile/utils/image/getImageDimensions.js
+packages/app-mobile/utils/image/resizeImage.js
 packages/app-mobile/utils/initializeCommandService.js
 packages/app-mobile/utils/ipc/RNToWebViewMessenger.js
 packages/app-mobile/utils/ipc/WebViewToRNMessenger.js
@@ -774,6 +776,7 @@ packages/editor/CodeMirror/utils/formatting/toggleRegionFormatGlobally.js
 packages/editor/CodeMirror/utils/formatting/toggleSelectedLinesStartWith.js
 packages/editor/CodeMirror/utils/formatting/types.js
 packages/editor/CodeMirror/utils/growSelectionToNode.js
+packages/editor/CodeMirror/utils/handlePasteEvent.js
 packages/editor/CodeMirror/utils/isInSyntaxNode.js
 packages/editor/CodeMirror/utils/setupVim.js
 packages/editor/SelectionFormatting.js

packages/app-desktop/gui/NoteEditor/NoteBody/CodeMirror/v6/CodeMirror.tsx

@@ -385,6 +385,7 @@ const CodeMirror = (props: NoteBodyEditorProps, ref: ForwardedRef<NoteBodyEditor
 					ref={editorRef}
 					settings={editorSettings}
 					pluginStates={props.plugins}
+					onPasteFile={null}
 					onEvent={onEditorEvent}
 					onLogMessage={logDebug}
 					onEditorPaste={onEditorPaste}

packages/app-mobile/components/NoteEditor/CodeMirror/CodeMirror.ts

@@ -27,6 +27,21 @@ export const initCodeMirror = (
 		initialText,
 		settings,
 
+		onPasteFile: async (data) => {
+			const reader = new FileReader();
+			return new Promise<void>((resolve, reject) => {
+				reader.onload = async () => {
+					const dataUrl = reader.result as string;
+					const base64 = dataUrl.replace(/^data:.*;base64,/, '');
+					await messenger.remoteApi.onPasteFile(data.type, base64);
+					resolve();
+				};
+				reader.onerror = () => reject(new Error('Failed to load file.'));
+
+				reader.readAsDataURL(data);
+			});
+		},
+
 		onLogMessage: message => {
 			void messenger.remoteApi.logMessage(message);
 		},

packages/app-mobile/components/NoteEditor/NoteEditor.test.tsx

@@ -38,7 +38,7 @@ describe('NoteEditor', () => {
 					onChange={()=>{}}
 					onSelectionChange={()=>{}}
 					onUndoRedoDepthChange={()=>{}}
-					onAttach={()=>{}}
+					onAttach={async ()=>{}}
 					plugins={{}}
 				/>
 			</MenuProvider>,

packages/app-mobile/components/NoteEditor/NoteEditor.tsx

@@ -26,11 +26,14 @@ import { WebViewErrorEvent } from 'react-native-webview/lib/RNCWebViewNativeComp
 import Logger from '@joplin/utils/Logger';
 import { PluginStates } from '@joplin/lib/services/plugins/reducer';
 import useEditorCommandHandler from './hooks/useEditorCommandHandler';
+import { join, dirname } from 'path';
+import * as mimeUtils from '@joplin/lib/mime-utils';
+import uuid from '@joplin/lib/uuid';
 
 type ChangeEventHandler = (event: ChangeEvent)=> void;
 type UndoRedoDepthChangeHandler = (event: UndoRedoDepthChangeEvent)=> void;
 type SelectionChangeEventHandler = (event: SelectionRangeChangeEvent)=> void;
-type OnAttachCallback = ()=> void;
+type OnAttachCallback = (filePath?: string)=> Promise<void>;
 
 const logger = Logger.create('NoteEditor');
 
@@ -373,6 +376,9 @@ function NoteEditor(props: Props, ref: any) {
 
 	const onEditorEvent = useRef((_event: EditorEvent) => {});
 
+	const onAttachRef = useRef(props.onAttach);
+	onAttachRef.current = props.onAttach;
+
 	const editorMessenger = useMemo(() => {
 		const localApi: WebViewToEditorApi = {
 			async onEditorEvent(event) {
@@ -381,6 +387,16 @@ function NoteEditor(props: Props, ref: any) {
 			async logMessage(message) {
 				logger.debug('CodeMirror:', message);
 			},
+			async onPasteFile(type, data) {
+				const tempFilePath = join(Setting.value('tempDir'), `paste.${uuid.createNano()}.${mimeUtils.toFileExtension(type)}`);
+				await shim.fsDriver().mkdir(dirname(tempFilePath));
+				try {
+					await shim.fsDriver().writeFile(tempFilePath, data, 'base64');
+					await onAttachRef.current(tempFilePath);
+				} finally {
+					await shim.fsDriver().remove(tempFilePath);
+				}
+			},
 		};
 		const messenger = new RNToWebViewMessenger<WebViewToEditorApi, EditorBodyControl>(
 			'editor', webviewRef, localApi,

packages/app-mobile/components/NoteEditor/types.ts

@@ -57,4 +57,5 @@ export interface SelectionRange {
 export interface WebViewToEditorApi {
 	onEditorEvent(event: EditorEvent): Promise<void>;
 	logMessage(message: string): Promise<void>;
+	onPasteFile(type: string, dataBase64: string): Promise<void>;
 }

packages/app-mobile/components/screens/Note.tsx

@@ -6,10 +6,9 @@ import UndoRedoService from '@joplin/lib/services/UndoRedoService';
 import NoteBodyViewer from '../NoteBodyViewer/NoteBodyViewer';
 import checkPermissions from '../../utils/checkPermissions';
 import NoteEditor from '../NoteEditor/NoteEditor';
-import { Size } from '@joplin/utils/types';
 const FileViewer = require('react-native-file-viewer').default;
 const React = require('react');
-import { Keyboard, View, TextInput, StyleSheet, Linking, Image, Share, NativeSyntheticEvent } from 'react-native';
+import { Keyboard, View, TextInput, StyleSheet, Linking, Share, NativeSyntheticEvent } from 'react-native';
 import { Platform, PermissionsAndroid } from 'react-native';
 const { connect } = require('react-redux');
 // const { MarkdownEditor } = require('@joplin/lib/../MarkdownEditor/index.js');
@@ -36,7 +35,6 @@ import { BaseScreenComponent } from '../base-screen';
 import { themeStyle, editorFont } from '../global-style';
 const { dialogs } = require('../../utils/dialogs.js');
 const DialogBox = require('react-native-dialogbox').default;
-import ImageResizer from '@bam.tech/react-native-image-resizer';
 import shared, { BaseNoteScreenComponent } from '@joplin/lib/components/shared/note-screen-shared';
 import { Asset, ImagePickerResponse, launchImageLibrary } from 'react-native-image-picker';
 import SelectDateTimeDialog from '../SelectDateTimeDialog';
@@ -65,6 +63,8 @@ import debounce from '../../utils/debounce';
 import { focus } from '@joplin/lib/utils/focusHandler';
 import CommandService from '@joplin/lib/services/CommandService';
 import * as urlUtils from '@joplin/lib/urlUtils';
+import getImageDimensions from '../../utils/image/getImageDimensions';
+import resizeImage from '../../utils/image/resizeImage';
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied
 const emptyArray: any[] = [];
@@ -682,24 +682,9 @@ class NoteScreenComponent extends BaseScreenComponent<Props, State> implements B
 		return result;
 	}
 
-	public async imageDimensions(uri: string): Promise<Size> {
-		return new Promise((resolve, reject) => {
-			Image.getSize(
-				uri,
-				(width: number, height: number) => {
-					resolve({ width: width, height: height });
-				},
-				// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied
-				(error: any) => {
-					reject(error);
-				},
-			);
-		});
-	}
-
 	public async resizeImage(localFilePath: string, targetPath: string, mimeType: string) {
 		const maxSize = Resource.IMAGE_MAX_DIMENSION;
-		const dimensions = await this.imageDimensions(localFilePath);
+		const dimensions = await getImageDimensions(localFilePath);
 		reg.logger().info('Original dimensions ', dimensions);
 
 		const saveOriginalImage = async () => {
@@ -711,30 +696,14 @@ class NoteScreenComponent extends BaseScreenComponent<Props, State> implements B
 			dimensions.height = maxSize;
 			reg.logger().info('New dimensions ', dimensions);
 
-			const format = mimeType === 'image/png' ? 'PNG' : 'JPEG';
+			await resizeImage({
-			reg.logger().info(`Resizing image ${localFilePath}`);
+				inputPath: localFilePath,
-			const resizedImage = await ImageResizer.createResizedImage(
+				outputPath: targetPath,
-				localFilePath,
+				maxWidth: dimensions.width,
-				dimensions.width,
+				maxHeight: dimensions.height,
-				dimensions.height,
+				quality: 85,
-				format,
+				format: mimeType === 'image/png' ? 'PNG' : 'JPEG',
-				85, // quality
+			});
-				undefined, // rotation
-				undefined, // outputPath
-				true, // keep metadata
-			);
-
-			const resizedImagePath = resizedImage.uri;
-			reg.logger().info('Resized image ', resizedImagePath);
-			reg.logger().info(`Moving ${resizedImagePath} => ${targetPath}`);
-
-			await shim.fsDriver().copy(resizedImagePath, targetPath);
-
-			try {
-				await shim.fsDriver().unlink(resizedImagePath);
-			} catch (error) {
-				reg.logger().warn('Error when unlinking cached file: ', error);
-			}
 			return true;
 		};
 
@@ -1140,11 +1109,19 @@ class NoteScreenComponent extends BaseScreenComponent<Props, State> implements B
 
 		const buttonId = await dialogs.pop(this, _('Choose an option'), buttons);
 
-		if (buttonId === 'takePhoto') this.takePhoto_onPress();
+		if (buttonId === 'takePhoto') await this.takePhoto_onPress();
-		if (buttonId === 'attachFile') void this.attachFile_onPress();
+		if (buttonId === 'attachFile') await this.attachFile_onPress();
-		if (buttonId === 'attachPhoto') void this.attachPhoto_onPress();
+		if (buttonId === 'attachPhoto') await this.attachPhoto_onPress();
 	}
 
+	public onAttach = async (filePath?: string) => {
+		if (filePath) {
+			await this.attachFile({ uri: filePath }, 'all');
+		} else {
+			await this.showAttachMenu();
+		}
+	};
+
 	// private vosk_:Vosk;
 
 	// private async getVosk() {
@@ -1585,7 +1562,7 @@ class NoteScreenComponent extends BaseScreenComponent<Props, State> implements B
 					onChange={this.onMarkdownEditorTextChange}
 					onSelectionChange={this.onMarkdownEditorSelectionChange}
 					onUndoRedoDepthChange={this.onUndoRedoDepthChange}
-					onAttach={() => this.showAttachMenu()}
+					onAttach={this.onAttach}
 					readOnly={this.state.readOnly}
 					plugins={this.props.plugins}
 					style={{

packages/app-mobile/utils/image/getImageDimensions.ts

@@ -0,0 +1,18 @@
+import { Size } from '@joplin/utils/types';
+import { Image as NativeImage } from 'react-native';
+
+const getImageDimensions = async (uri: string): Promise<Size> => {
+	return new Promise((resolve, reject) => {
+		NativeImage.getSize(
+			uri,
+			(width: number, height: number) => {
+				resolve({ width: width, height: height });
+			},
+			(error: unknown) => {
+				reject(error);
+			},
+		);
+	});
+};
+
+export default getImageDimensions;

packages/app-mobile/utils/image/resizeImage.ts

@@ -0,0 +1,43 @@
+import shim from '@joplin/lib/shim';
+import Logger from '@joplin/utils/Logger';
+import ImageResizer from '@bam.tech/react-native-image-resizer';
+
+const logger = Logger.create('resizeImage');
+
+type OutputFormat = 'PNG' | 'JPEG';
+
+interface Options {
+	inputPath: string;
+	outputPath: string;
+	maxWidth: number;
+	maxHeight: number;
+	format: OutputFormat;
+	quality: number;
+}
+
+const resizeImage = async (options: Options) => {
+	const resizedImage = await ImageResizer.createResizedImage(
+		options.inputPath,
+		options.maxWidth,
+		options.maxHeight,
+		options.format,
+		options.quality, // quality
+		undefined, // rotation
+		undefined, // outputPath
+		true, // keep metadata
+	);
+
+	const resizedImagePath = resizedImage.uri;
+	logger.info('Resized image ', resizedImagePath);
+	logger.info(`Moving ${resizedImagePath} => ${options.outputPath}`);
+
+	await shim.fsDriver().copy(resizedImagePath, options.outputPath);
+
+	try {
+		await shim.fsDriver().unlink(resizedImagePath);
+	} catch (error) {
+		logger.warn('Error when unlinking cached file: ', error);
+	}
+};
+
+export default resizeImage;

packages/editor/CodeMirror/createEditor.test.ts

@@ -39,6 +39,7 @@ describe('createEditor', () => {
 			settings: editorSettings,
 			onEvent: _event => {},
 			onLogMessage: _message => {},
+			onPasteFile: null,
 		});
 
 		// Force the generation of the syntax tree now.
@@ -66,6 +67,7 @@ describe('createEditor', () => {
 			settings: editorSettings,
 			onEvent: _event => {},
 			onLogMessage: _message => {},
+			onPasteFile: null,
 		});
 
 		const getContentScriptJs = jest.fn(async () => {
@@ -133,6 +135,7 @@ describe('createEditor', () => {
 			settings: editorSettings,
 			onEvent: _event => {},
 			onLogMessage: _message => {},
+			onPasteFile: null,
 		});
 
 		const getContentScriptJs = jest.fn(async () => {

packages/editor/CodeMirror/createEditor.ts

@@ -30,6 +30,7 @@ import configFromSettings from './configFromSettings';
 import getScrollFraction from './getScrollFraction';
 import CodeMirrorControl from './CodeMirrorControl';
 import insertLineAfter from './editorCommands/insertLineAfter';
+import handlePasteEvent from './utils/handlePasteEvent';
 
 const createEditor = (
 	parentElement: HTMLElement, props: EditorProps,
@@ -257,6 +258,24 @@ const createEditor = (
 							fraction: getScrollFraction(view),
 						});
 					},
+					paste: (event, view) => {
+						if (props.onPasteFile) {
+							handlePasteEvent(event, view, props.onPasteFile);
+						}
+					},
+					dragover: (event, _view) => {
+						if (props.onPasteFile && event.dataTransfer.files.length) {
+							event.preventDefault();
+							event.dataTransfer.dropEffect = 'copy';
+							return true;
+						}
+						return false;
+					},
+					drop: (event, view) => {
+						if (props.onPasteFile) {
+							handlePasteEvent(event, view, props.onPasteFile);
+						}
+					},
 				}),
 
 				EditorState.tabSize.of(4),

packages/editor/CodeMirror/testUtil/createEditorControl.ts

@@ -10,6 +10,7 @@ const createEditorControl = (initialText: string) => {
 		settings: editorSettings,
 		onEvent: _event => {},
 		onLogMessage: _message => {},
+		onPasteFile: null,
 	});
 };
 

packages/editor/CodeMirror/utils/handlePasteEvent.ts

@@ -0,0 +1,29 @@
+import { EditorView } from '@codemirror/view';
+import { PasteFileCallback } from '../../types';
+
+const handlePasteEvent = (event: ClipboardEvent|DragEvent, _view: EditorView, onPaste: PasteFileCallback) => {
+	const dataTransfer = 'clipboardData' in event ? event.clipboardData : event.dataTransfer;
+	const files = dataTransfer.files;
+
+	let fileToPaste: File|null = null;
+
+	// Prefer image files, if available.
+	for (const file of files) {
+		if (['image/png', 'image/jpeg', 'image/svg+xml'].includes(file.type)) {
+			fileToPaste = file;
+			break;
+		}
+	}
+
+	// Fall back to other files
+	if (files.length && !fileToPaste) {
+		fileToPaste = files[0];
+	}
+
+	if (fileToPaste) {
+		event.preventDefault();
+		void onPaste(fileToPaste);
+	}
+};
+
+export default handlePasteEvent;

packages/editor/types.ts

@@ -168,11 +168,14 @@ export interface EditorSettings {
 
 export type LogMessageCallback = (message: string)=> void;
 export type OnEventCallback = (event: EditorEvent)=> void;
+export type PasteFileCallback = (data: File)=> Promise<void>;
 
 export interface EditorProps {
 	settings: EditorSettings;
 	initialText: string;
 
+	// If null, paste and drag-and-drop will not work for resources unless handled elsewhere.
+	onPasteFile: PasteFileCallback|null;
 	onEvent: OnEventCallback;
 	onLogMessage: LogMessageCallback;
 }