Desktop: Resolves #279: Add support for editable resources (#3305)

* Add support for editable resources

* Fixed handling of resource paths

* Add delay before saving resource

* Delete temp files and stop watching when switching notes

* Handle sync when resource has been edited

* Handle resource conflicts

* Refactoring and handle conflicts from edit watcher

* Added test unit for conflict handling when deleting resource

* Trying to fix Travis-ci test units

* Fixed typo

* Update TinyMCE.tsx

* Update AsyncActionQueue.ts

.eslintignore

@@ -97,6 +97,7 @@ ReactNativeClient/lib/joplin-renderer/MdToHtml/rules/fence.js
 ReactNativeClient/lib/joplin-renderer/MdToHtml/rules/mermaid.js
 ReactNativeClient/lib/joplin-renderer/MdToHtml/rules/sanitize_html.js
 ReactNativeClient/lib/JoplinServerApi.js
+ReactNativeClient/lib/services/ResourceEditWatcher.js
 ReactNativeClient/PluginAssetsLoader.js
 ReactNativeClient/setUpQuickActions.js
 # AUTO-GENERATED - EXCLUDED TYPESCRIPT BUILD

.gitignore

@@ -87,6 +87,7 @@ ReactNativeClient/lib/joplin-renderer/MdToHtml/rules/fence.js
 ReactNativeClient/lib/joplin-renderer/MdToHtml/rules/mermaid.js
 ReactNativeClient/lib/joplin-renderer/MdToHtml/rules/sanitize_html.js
 ReactNativeClient/lib/JoplinServerApi.js
+ReactNativeClient/lib/services/ResourceEditWatcher.js
 ReactNativeClient/PluginAssetsLoader.js
 ReactNativeClient/setUpQuickActions.js
 # AUTO-GENERATED - EXCLUDED TYPESCRIPT BUILD

CliClient/tests/models_Note.js

@@ -223,6 +223,8 @@ describe('models_Note', function() {
 		const resourceDir = Setting.value('resourceDir');
 		const r1 = await shim.createResourceFromPath(`${__dirname}/../tests/support/photo.jpg`);
 		const r2 = await shim.createResourceFromPath(`${__dirname}/../tests/support/photo.jpg`);
+		const t1 = r1.updated_time;
+		const t2 = r2.updated_time;
 
 		const testCases = [
 			[
@@ -248,12 +250,12 @@ describe('models_Note', function() {
 			[
 				true,
 				`![](:/${r1.id})`,
-				`![](file://${resourceDir}/${r1.id}.jpg)`,
+				`![](file://${resourceDir}/${r1.id}.jpg?t=${t1})`,
 			],
 			[
 				true,
 				`![](:/${r1.id}) ![](:/${r1.id}) ![](:/${r2.id})`,
-				`![](file://${resourceDir}/${r1.id}.jpg) ![](file://${resourceDir}/${r1.id}.jpg) ![](file://${resourceDir}/${r2.id}.jpg)`,
+				`![](file://${resourceDir}/${r1.id}.jpg?t=${t1}) ![](file://${resourceDir}/${r1.id}.jpg?t=${t1}) ![](file://${resourceDir}/${r2.id}.jpg?t=${t2})`,
 			],
 		];
 

CliClient/tests/synchronizer.js

@@ -3,7 +3,7 @@
 require('app-module-path').addPath(__dirname);
 
 const { time } = require('lib/time-utils.js');
-const { setupDatabase, allSyncTargetItemsEncrypted, kvStore, revisionService, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient, syncTargetId, encryptionService, loadEncryptionMasterKey, fileContentEqual, decryptionWorker, checkThrowAsync, asyncTest } = require('test-utils.js');
+const { setupDatabase, allSyncTargetItemsEncrypted, tempFilePath, resourceFetcher, kvStore, revisionService, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient, syncTargetId, encryptionService, loadEncryptionMasterKey, fileContentEqual, decryptionWorker, checkThrowAsync, asyncTest } = require('test-utils.js');
 const { shim } = require('lib/shim.js');
 const fs = require('fs-extra');
 const Folder = require('lib/models/Folder.js');
@@ -991,6 +991,145 @@ describe('synchronizer', function() {
 		expect(fileContentEqual(resourcePath1, resourcePath1_2)).toBe(true);
 	}));
 
+	it('should sync resource blob changes', asyncTest(async () => {
+		const tempFile = tempFilePath('txt');
+		await shim.fsDriver().writeFile(tempFile, '1234', 'utf8');
+		const folder1 = await Folder.save({ title: 'folder1' });
+		const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id });
+		await shim.attachFileToNote(note1, tempFile);
+		await synchronizer().start();
+
+		await switchClient(2);
+
+		await synchronizer().start();
+		await resourceFetcher().start();
+		await resourceFetcher().waitForAllFinished();
+		let resource1_2 = (await Resource.all())[0];
+		const modFile = tempFilePath('txt');
+		await shim.fsDriver().writeFile(modFile, '1234 MOD', 'utf8');
+		await Resource.updateResourceBlobContent(resource1_2.id, modFile);
+		const originalSize = resource1_2.size;
+		resource1_2 = (await Resource.all())[0];
+		const newSize = resource1_2.size;
+		expect(originalSize).toBe(4);
+		expect(newSize).toBe(8);
+		await synchronizer().start();
+
+		await switchClient(1);
+
+		await synchronizer().start();
+		await resourceFetcher().start();
+		await resourceFetcher().waitForAllFinished();
+		const resource1_1 = (await Resource.all())[0];
+		expect(resource1_1.size).toBe(newSize);
+		expect(await Resource.resourceBlobContent(resource1_1.id, 'utf8')).toBe('1234 MOD');
+	}));
+
+	it('should handle resource conflicts', asyncTest(async () => {
+		{
+			const tempFile = tempFilePath('txt');
+			await shim.fsDriver().writeFile(tempFile, '1234', 'utf8');
+			const folder1 = await Folder.save({ title: 'folder1' });
+			const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id });
+			await shim.attachFileToNote(note1, tempFile);
+			await synchronizer().start();
+		}
+
+		await switchClient(2);
+
+		{
+			await synchronizer().start();
+			await resourceFetcher().start();
+			await resourceFetcher().waitForAllFinished();
+			const resource = (await Resource.all())[0];
+			const modFile2 = tempFilePath('txt');
+			await shim.fsDriver().writeFile(modFile2, '1234 MOD 2', 'utf8');
+			await Resource.updateResourceBlobContent(resource.id, modFile2);
+			await synchronizer().start();
+		}
+
+		await switchClient(1);
+
+		{
+			// Going to modify a resource without syncing first, which will cause a conflict
+			const resource = (await Resource.all())[0];
+			const modFile1 = tempFilePath('txt');
+			await shim.fsDriver().writeFile(modFile1, '1234 MOD 1', 'utf8');
+			await Resource.updateResourceBlobContent(resource.id, modFile1);
+			await synchronizer().start(); // CONFLICT
+
+			// If we try to read the resource content now, it should throw because the local
+			// content has been moved to the conflict notebook, and the new local content
+			// has not been downloaded yet.
+			await checkThrowAsync(async () => await Resource.resourceBlobContent(resource.id));
+
+			// Now download resources, and our local content would have been overwritten by
+			// the content from client 2
+			await resourceFetcher().start();
+			await resourceFetcher().waitForAllFinished();
+			const localContent =  await Resource.resourceBlobContent(resource.id, 'utf8');
+			expect(localContent).toBe('1234 MOD 2');
+
+			// Check that the Conflict note has been generated, with the conflict resource
+			// attached to it, and check that it has the original content.
+			const allNotes = await Note.all();
+			expect(allNotes.length).toBe(2);
+			const conflictNote = allNotes.find((v) => {
+				return !!v.is_conflict;
+			});
+			expect(!!conflictNote).toBe(true);
+			const resourceIds = await Note.linkedResourceIds(conflictNote.body);
+			expect(resourceIds.length).toBe(1);
+			const conflictContent =  await Resource.resourceBlobContent(resourceIds[0], 'utf8');
+			expect(conflictContent).toBe('1234 MOD 1');
+		}
+	}));
+
+	it('should handle resource conflicts if a resource is changed locally but deleted remotely', asyncTest(async () => {
+		{
+			const tempFile = tempFilePath('txt');
+			await shim.fsDriver().writeFile(tempFile, '1234', 'utf8');
+			const folder1 = await Folder.save({ title: 'folder1' });
+			const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id });
+			await shim.attachFileToNote(note1, tempFile);
+			await synchronizer().start();
+		}
+
+		await switchClient(2);
+
+		{
+			await synchronizer().start();
+			await resourceFetcher().start();
+			await resourceFetcher().waitForAllFinished();
+		}
+
+		await switchClient(1);
+
+		{
+			const resource = (await Resource.all())[0];
+			await Resource.delete(resource.id);
+			await synchronizer().start();
+
+		}
+
+		await switchClient(2);
+
+		{
+			const originalResource = (await Resource.all())[0];
+			await Resource.save({ id: originalResource.id, title: 'modified resource' });
+			await synchronizer().start(); // CONFLICT
+
+			const deletedResource = await Resource.load(originalResource.id);
+			expect(!deletedResource).toBe(true);
+
+			const allResources = await Resource.all();
+			expect(allResources.length).toBe(1);
+			const conflictResource = allResources[0];
+			expect(originalResource.id).not.toBe(conflictResource.id);
+			expect(conflictResource.title).toBe('modified resource');
+		}
+	}));
+
 	it('should upload decrypted items to sync target after encryption disabled', asyncTest(async () => {
 		Setting.setValue('encryption.enabled', true);
 		const masterKey = await loadEncryptionMasterKey();

CliClient/tests/test-utils.js

@@ -37,9 +37,11 @@ const EncryptionService = require('lib/services/EncryptionService.js');
 const DecryptionWorker = require('lib/services/DecryptionWorker.js');
 const ResourceService = require('lib/services/ResourceService.js');
 const RevisionService = require('lib/services/RevisionService.js');
+const ResourceFetcher = require('lib/services/ResourceFetcher.js');
 const KvStore = require('lib/services/KvStore.js');
 const WebDavApi = require('lib/WebDavApi');
 const DropboxApi = require('lib/DropboxApi');
+const md5 = require('md5');
 
 const databases_ = [];
 const synchronizers_ = [];
@@ -47,6 +49,7 @@ const encryptionServices_ = [];
 const revisionServices_ = [];
 const decryptionWorkers_ = [];
 const resourceServices_ = [];
+const resourceFetchers_ = [];
 const kvStores_ = [];
 let fileApi_ = null;
 let currentClient_ = 1;
@@ -250,6 +253,7 @@ async function setupDatabaseAndSynchronizer(id = null) {
 	decryptionWorkers_[id] = new DecryptionWorker();
 	decryptionWorkers_[id].setEncryptionService(encryptionServices_[id]);
 	resourceServices_[id] = new ResourceService();
+	resourceFetchers_[id] = new ResourceFetcher(() => { return synchronizers_[id].api(); });
 	kvStores_[id] = new KvStore();
 
 	await fileApi().clearRoot();
@@ -294,6 +298,11 @@ function resourceService(id = null) {
 	return resourceServices_[id];
 }
 
+function resourceFetcher(id = null) {
+	if (id === null) id = currentClient_;
+	return resourceFetchers_[id];
+}
+
 async function loadEncryptionMasterKey(id = null, useExisting = false) {
 	const service = encryptionService(id);
 
@@ -478,6 +487,10 @@ async function createNTestTags(n) {
 	return tags;
 }
 
+function tempFilePath(ext) {
+	return `${Setting.value('tempDir')}/${md5(Date.now() + Math.random())}.${ext}`;
+}
+
 // Application for feature integration testing
 class TestApp extends BaseApplication {
 	constructor(hasGui = true) {
@@ -546,4 +559,4 @@ class TestApp extends BaseApplication {
 	}
 }
 
-module.exports = { kvStore, resourceService, allSyncTargetItemsEncrypted, setupDatabase, revisionService, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient, syncTargetId, objectsEqual, checkThrowAsync, encryptionService, loadEncryptionMasterKey, fileContentEqual, decryptionWorker, asyncTest, currentClientId, id, ids, sortedIds, at, createNTestNotes, createNTestFolders, createNTestTags, TestApp };
+module.exports = { kvStore, resourceService, resourceFetcher, tempFilePath, allSyncTargetItemsEncrypted, setupDatabase, revisionService, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient, syncTargetId, objectsEqual, checkThrowAsync, encryptionService, loadEncryptionMasterKey, fileContentEqual, decryptionWorker, asyncTest, currentClientId, id, ids, sortedIds, at, createNTestNotes, createNTestFolders, createNTestTags, TestApp };

ElectronClient/app.js

@@ -23,6 +23,7 @@ const InteropServiceHelper = require('./InteropServiceHelper.js');
 const ResourceService = require('lib/services/ResourceService');
 const ClipperServer = require('lib/ClipperServer');
 const ExternalEditWatcher = require('lib/services/ExternalEditWatcher');
+const ResourceEditWatcher = require('lib/services/ResourceEditWatcher').default;
 const { bridge } = require('electron').remote.require('./bridge');
 const { shell, webFrame, clipboard } = require('electron');
 const Menu = bridge().Menu;
@@ -1505,6 +1506,8 @@ class Application extends BaseApplication {
 		ExternalEditWatcher.instance().setLogger(reg.logger());
 		ExternalEditWatcher.instance().dispatch = this.store().dispatch;
 
+		ResourceEditWatcher.instance().initialize(reg.logger(), this.store().dispatch);
+
 		RevisionService.instance().runInBackground();
 
 		this.updateMenuItemStates();

ElectronClient/gui/NoteEditor/NoteBody/TinyMCE/TinyMCE.tsx

@@ -158,7 +158,10 @@ const TinyMCE = (props:NoteBodyEditorProps, ref:any) => {
 	const markupToHtml = useRef(null);
 	markupToHtml.current = props.markupToHtml;
 
-	const lastOnChangeEventContent = useRef<string>('');
+	const lastOnChangeEventInfo = useRef<any>({
+		content: null,
+		resourceInfos: null,
+	});
 
 	const rootIdRef = useRef<string>(`tinymce-${Date.now()}${Math.round(Math.random() * 10000)}`);
 	const editorRef = useRef<any>(null);
@@ -761,10 +764,15 @@ const TinyMCE = (props:NoteBodyEditorProps, ref:any) => {
 		let cancelled = false;
 
 		const loadContent = async () => {
-			if (lastOnChangeEventContent.current !== props.content) {
+			if (lastOnChangeEventInfo.current.content !== props.content || lastOnChangeEventInfo.current.resourceInfos !== props.resourceInfos) {
 				const result = await props.markupToHtml(props.contentMarkupLanguage, props.content, markupRenderOptions({ resourceInfos: props.resourceInfos }));
 				if (cancelled) return;
-				lastOnChangeEventContent.current = props.content;
+
+				lastOnChangeEventInfo.current = {
+					content: props.content,
+					resourceInfos: props.resourceInfos,
+				};
+
 				editor.setContent(result.html);
 			}
 
@@ -859,7 +867,7 @@ const TinyMCE = (props:NoteBodyEditorProps, ref:any) => {
 
 				if (!editor) return;
 
-				lastOnChangeEventContent.current = contentMd;
+				lastOnChangeEventInfo.current.content = contentMd;
 
 				props_onChangeRef.current({
 					changeId: changeId,

ElectronClient/gui/NoteEditor/NoteEditor.tsx

@@ -28,6 +28,7 @@ const { _ } = require('lib/locale');
 const Note = require('lib/models/Note.js');
 const { bridge } = require('electron').remote.require('./bridge');
 const ExternalEditWatcher = require('lib/services/ExternalEditWatcher');
+import ResourceEditWatcher from '../../lib/services/ResourceEditWatcher';
 const eventManager = require('../../eventManager');
 const NoteRevisionViewer = require('../NoteRevisionViewer.min');
 const TagList = require('../TagList.min.js');
@@ -172,6 +173,8 @@ function NoteEditor(props: NoteEditorProps) {
 			type: props.selectedNoteHash ? ScrollOptionTypes.Hash : ScrollOptionTypes.Percent,
 			value: props.selectedNoteHash ? props.selectedNoteHash : props.lastEditorScrollPercents[props.noteId] || 0,
 		});
+
+		ResourceEditWatcher.instance().stopWatchingAll();
 	}, [formNote.id, previousNoteId]);
 
 	const onFieldChange = useCallback((field: string, value: any, changeId = 0) => {

ElectronClient/gui/NoteEditor/utils/contextMenu.ts

@@ -7,6 +7,8 @@ const { clipboard } = require('electron');
 const { toSystemSlashes } = require('lib/path-utils');
 const { _ } = require('lib/locale');
 
+import ResourceEditWatcher from '../../../lib/services/ResourceEditWatcher';
+
 export enum ContextMenuItemType {
 	None = '',
 	Image = 'image',
@@ -42,9 +44,12 @@ export function menuItems():ContextMenuItems {
 		open: {
 			label: _('Open...'),
 			onAction: async (options:ContextMenuOptions) => {
-				const { resourcePath } = await resourceInfo(options);
-				const ok = bridge().openExternal(`file://${resourcePath}`);
-				if (!ok) bridge().showErrorMessageBox(_('This file could not be opened: %s', resourcePath));
+				try {
+					await ResourceEditWatcher.instance().openAndWatch(options.resourceId);
+				} catch (error) {
+					console.error(error);
+					bridge().showErrorMessageBox(error.message);
+				}
 			},
 			isActive: (itemType:ContextMenuItemType) => itemType === ContextMenuItemType.Image || itemType === ContextMenuItemType.Resource,
 		},

ElectronClient/gui/NoteEditor/utils/useFormNote.ts

@@ -11,6 +11,7 @@ const Setting = require('lib/models/Setting');
 const { reg } = require('lib/registry.js');
 const ResourceFetcher = require('lib/services/ResourceFetcher.js');
 const DecryptionWorker = require('lib/services/DecryptionWorker.js');
+const ResourceEditWatcher = require('lib/services/ResourceEditWatcher.js').default;
 
 export interface OnLoadEvent {
 	formNote: FormNote,
@@ -30,12 +31,14 @@ function installResourceChangeHandler(onResourceChangeHandler: Function) {
 	ResourceFetcher.instance().on('downloadComplete', onResourceChangeHandler);
 	ResourceFetcher.instance().on('downloadStarted', onResourceChangeHandler);
 	DecryptionWorker.instance().on('resourceDecrypted', onResourceChangeHandler);
+	ResourceEditWatcher.instance().on('resourceChange', onResourceChangeHandler);
 }
 
 function uninstallResourceChangeHandler(onResourceChangeHandler: Function) {
 	ResourceFetcher.instance().off('downloadComplete', onResourceChangeHandler);
 	ResourceFetcher.instance().off('downloadStarted', onResourceChangeHandler);
 	DecryptionWorker.instance().off('resourceDecrypted', onResourceChangeHandler);
+	ResourceEditWatcher.instance().off('resourceChange', onResourceChangeHandler);
 }
 
 export default function useFormNote(dependencies:HookDependencies) {

ElectronClient/gui/NoteEditor/utils/useMessageHandler.ts

@@ -10,6 +10,7 @@ const { urlDecode } = require('lib/string-utils');
 const urlUtils = require('lib/urlUtils');
 const ResourceFetcher = require('lib/services/ResourceFetcher.js');
 const { reg } = require('lib/registry.js');
+import ResourceEditWatcher from '../../../lib/services/ResourceEditWatcher';
 
 export default function useMessageHandler(scrollWhenReady:any, setScrollWhenReady:Function, editorRef:any, setLocalSearchResultCount:Function, dispatch:Function, formNote:FormNote) {
 	return useCallback(async (event: any) => {
@@ -17,7 +18,7 @@ export default function useMessageHandler(scrollWhenReady:any, setScrollWhenRead
 		const args = event.args;
 		const arg0 = args && args.length >= 1 ? args[0] : null;
 
-		if (msg !== 'percentScroll') console.info(`Got ipc-message: ${msg}`, args);
+		if (msg !== 'percentScroll') console.info(`Got ipc-message: ${msg}`, arg0);
 
 		if (msg.indexOf('error:') === 0) {
 			const s = msg.split(':');
@@ -60,8 +61,13 @@ export default function useMessageHandler(scrollWhenReady:any, setScrollWhenRead
 					}
 					return;
 				}
-				const filePath = Resource.fullPath(item);
-				bridge().openItem(filePath);
+
+				try {
+					await ResourceEditWatcher.instance().openAndWatch(item.id);
+				} catch (error) {
+					console.error(error);
+					bridge().showErrorMessageBox(error.message);
+				}
 			} else if (item.type_ === BaseModel.TYPE_NOTE) {
 				dispatch({
 					type: 'FOLDER_AND_NOTE_SELECT',

README.md

@@ -69,6 +69,7 @@ The Web Clipper is a browser extension that allows you to save web pages and scr
 	- [Joplin Forum](https://discourse.joplinapp.org)
 	- [Markdown Guide](https://github.com/laurent22/joplin/blob/master/readme/markdown.md)
 	- [How to enable end-to-end encryption](https://github.com/laurent22/joplin/blob/master/readme/e2ee.md)
+	- [What is a conflict?](https://github.com/laurent22/joplin/blob/master/readme/conflict.md)
 	- [End-to-end encryption spec](https://github.com/laurent22/joplin/blob/master/readme/spec.md)
 	- [How to enable debug mode](https://github.com/laurent22/joplin/blob/master/readme/debugging.md)
 	- [API documentation](https://github.com/laurent22/joplin/blob/master/readme/api.md)

ReactNativeClient/lib/AsyncActionQueue.ts

@@ -7,6 +7,10 @@ export interface QueueItem {
 	context: any,
 }
 
+// The AsyncActionQueue can be used to debounce asynchronous actions, to make sure
+// they run in the right order, and also to ensure that if multiple actions are emitted
+// only the last one is executed. This is particularly useful to save data in the background.
+// Each queue should be associated with a specific entity (a note, resource, etc.)
 export default class AsyncActionQueue {
 
 	queue_:QueueItem[] = [];
@@ -64,6 +68,8 @@ export default class AsyncActionQueue {
 	}
 
 	waitForAllDone() {
+		if (!this.queue_.length) return Promise.resolve();
+
 		this.scheduleProcessing(1);
 
 		return new Promise((resolve) => {

ReactNativeClient/lib/BaseApplication.js

@@ -521,7 +521,7 @@ class BaseApplication {
 			DecryptionWorker.instance().scheduleStart();
 		}
 
-		if (this.hasGui() && action.type === 'SYNC_CREATED_RESOURCE') {
+		if (this.hasGui() && action.type === 'SYNC_CREATED_OR_UPDATED_RESOURCE') {
 			ResourceFetcher.instance().autoAddResources();
 		}
 

ReactNativeClient/lib/joplin-renderer/MdToHtml/rules/link_open.js

@@ -23,8 +23,9 @@ function installRule(markdownIt, mdOptions, ruleOptions) {
 		let icon = '';
 		let hrefAttr = '#';
 		let mime = '';
+		let resourceId = '';
 		if (isResourceUrl) {
-			const resourceId = resourceHrefInfo.itemId;
+			resourceId = resourceHrefInfo.itemId;
 
 			const result = ruleOptions.resources[resourceId];
 			const resourceStatus = utils.resourceStatus(ruleOptions.ResourceModel, result);
@@ -62,7 +63,7 @@ function installRule(markdownIt, mdOptions, ruleOptions) {
 		// https://github.com/laurent22/joplin/issues/2030
 		href = href.replace(/'/g, '%27');
 
-		let js = `${ruleOptions.postMessageSyntax}(${JSON.stringify(href)}); return false;`;
+		let js = `${ruleOptions.postMessageSyntax}(${JSON.stringify(href)}, { resourceId: ${JSON.stringify(resourceId)} }); return false;`;
 		if (hrefAttr.indexOf('#') === 0 && href.indexOf('#') === 0) js = ''; // If it's an internal anchor, don't add any JS since the webview is going to handle navigating to the right place
 
 		if (ruleOptions.plainResourceRendering || pluginOptions.linkRenderingType === 2) {

ReactNativeClient/lib/joplin-renderer/utils.js

@@ -141,6 +141,7 @@ utils.imageReplacement = function(ResourceModel, src, resources, resourceBaseUrl
 	if (ResourceModel.isSupportedImageMimeType(mime)) {
 		let newSrc = `./${ResourceModel.filename(resource)}`;
 		if (resourceBaseUrl) newSrc = resourceBaseUrl + newSrc;
+		newSrc += `?t=${resource.updated_time}`;
 		return {
 			'data-resource-id': resource.id,
 			src: newSrc,

ReactNativeClient/lib/models/Note.js

@@ -149,7 +149,7 @@ class Note extends BaseItem {
 			const id = resourceIds[i];
 			const resource = await Resource.load(id);
 			if (!resource) continue;
-			const resourcePath = options.useAbsolutePaths ? `file://${Resource.fullPath(resource)}` : Resource.relativePath(resource);
+			const resourcePath = options.useAbsolutePaths ? `${`file://${Resource.fullPath(resource)}` + '?t='}${resource.updated_time}` : Resource.relativePath(resource);
 			body = body.replace(new RegExp(`:/${id}`, 'gi'), markdownUtils.escapeLinkUrl(resourcePath));
 		}
 
@@ -174,12 +174,19 @@ class Note extends BaseItem {
 		this.logger().info('replaceResourceExternalToInternalLinks', 'options:', options, 'pathsToTry:', pathsToTry, 'body:', body);
 
 		for (const basePath of pathsToTry) {
-			const reString = `${pregQuote(`${basePath}/`)}[a-zA-Z0-9.]+`;
-			const re = new RegExp(reString, 'gi');
-			body = body.replace(re, match => {
-				const id = Resource.pathToId(match);
-				return `:/${id}`;
-			});
+			const reStrings = [
+				// Handles file://path/to/abcdefg.jpg?t=12345678
+				`${pregQuote(`${basePath}/`)}[a-zA-Z0-9.]+\\?t=[0-9]+`,
+				// Handles file://path/to/abcdefg.jpg
+				`${pregQuote(`${basePath}/`)}[a-zA-Z0-9.]+`,
+			];
+			for (const reString of reStrings) {
+				const re = new RegExp(reString, 'gi');
+				body = body.replace(re, match => {
+					const id = Resource.pathToId(match);
+					return `:/${id}`;
+				});
+			}
 		}
 
 		this.logger().info('replaceResourceExternalToInternalLinks result', body);

ReactNativeClient/lib/models/Resource.js

@@ -1,5 +1,6 @@
 const BaseModel = require('lib/BaseModel.js');
 const BaseItem = require('lib/models/BaseItem.js');
+const ItemChange = require('lib/models/ItemChange.js');
 const NoteResource = require('lib/models/NoteResource.js');
 const ResourceLocalState = require('lib/models/ResourceLocalState.js');
 const Setting = require('lib/models/Setting.js');
@@ -67,6 +68,7 @@ class Resource extends BaseItem {
 		return Resource.fsDriver_;
 	}
 
+	// DEPRECATED IN FAVOUR OF friendlySafeFilename()
 	static friendlyFilename(resource) {
 		let output = safeFilename(resource.title); // Make sure not to allow spaces or any special characters as it's not supported in HTTP headers
 		if (!output) output = resource.id;
@@ -91,6 +93,15 @@ class Resource extends BaseItem {
 		return resource.id + extension;
 	}
 
+	static friendlySafeFilename(resource) {
+		let ext = resource.extension;
+		if (!ext) ext = resource.mime ? mime.toFileExtension(resource.mime) : '';
+		const safeExt = ext ? pathUtils.safeFileExtension(ext).toLowerCase() : '';
+		let title = resource.title ? resource.title : resource.id;
+		if (safeExt && pathUtils.fileExtension(title).toLowerCase() === safeExt) title = pathUtils.filename(title);
+		return pathUtils.friendlySafeFilename(title) + (safeExt ? `.${safeExt}` : '');
+	}
+
 	static relativePath(resource, encryptedBlob = false) {
 		return `${Setting.value('resourceDirName')}/${this.filename(resource, encryptedBlob)}`;
 	}
@@ -100,8 +111,21 @@ class Resource extends BaseItem {
 	}
 
 	static async isReady(resource) {
+		const r = await this.readyStatus(resource);
+		return r === 'ok';
+	}
+
+	static async readyStatus(resource) {
 		const ls = await this.localState(resource);
-		return resource && ls.fetch_status === Resource.FETCH_STATUS_DONE && !resource.encryption_blob_encrypted;
+		if (!resource) return 'notFound';
+		if (ls.fetch_status !== Resource.FETCH_STATUS_DONE) return 'notDownloaded';
+		if (resource.encryption_blob_encrypted) return 'encrypted';
+		return 'ok';
+	}
+
+	static async requireIsReady(resource) {
+		const readyStatus = await Resource.readyStatus(resource);
+		if (readyStatus !== 'ok') throw new Error(`Resource is not ready. Status: ${readyStatus}`);
 	}
 
 	// For resources, we need to decrypt the item (metadata) and the resource binary blob.
@@ -218,7 +242,7 @@ class Resource extends BaseItem {
 		return url.substr(2);
 	}
 
-	static localState(resourceOrId) {
+	static async localState(resourceOrId) {
 		return ResourceLocalState.byResourceId(typeof resourceOrId === 'object' ? resourceOrId.id : resourceOrId);
 	}
 
@@ -297,6 +321,59 @@ class Resource extends BaseItem {
 		throw new Error(`Invalid status: ${status}`);
 	}
 
+	static async updateResourceBlobContent(resourceId, newBlobFilePath) {
+		const resource = await Resource.load(resourceId);
+		await this.requireIsReady(resource);
+
+		const fileStat = await this.fsDriver().stat(newBlobFilePath);
+		await this.fsDriver().copy(newBlobFilePath, Resource.fullPath(resource));
+
+		return await Resource.save({
+			id: resource.id,
+			size: fileStat.size,
+		});
+	}
+
+	static async resourceBlobContent(resourceId, encoding = 'Buffer') {
+		const resource = await Resource.load(resourceId);
+		await this.requireIsReady(resource);
+		return await this.fsDriver().readFile(Resource.fullPath(resource), encoding);
+	}
+
+	static async duplicateResource(resourceId) {
+		const resource = await Resource.load(resourceId);
+		const localState = await Resource.localState(resource);
+
+		let newResource = { ...resource };
+		delete newResource.id;
+		newResource = await Resource.save(newResource);
+
+		const newLocalState = { ...localState };
+		newLocalState.resource_id = newResource.id;
+		delete newLocalState.id;
+
+		await Resource.setLocalState(newResource, newLocalState);
+
+		const sourcePath = Resource.fullPath(resource);
+		if (await this.fsDriver().exists(sourcePath)) {
+			await this.fsDriver().copy(sourcePath, Resource.fullPath(newResource));
+		}
+
+		return newResource;
+	}
+
+	static async createConflictResourceNote(resource) {
+		const Note = this.getClass('Note');
+
+		const conflictResource = await Resource.duplicateResource(resource.id);
+
+		await Note.save({
+			title: _('Attachment conflict: "%s"', resource.title),
+			body: _('There was a [conflict](%s) on the attachment below.\n\n%s', 'https://joplinapp.org/conflict', Resource.markdownTag(conflictResource)),
+			is_conflict: 1,
+		}, { changeSource: ItemChange.SOURCE_SYNC });
+	}
+
 }
 
 Resource.IMAGE_MAX_DIMENSION = 1920;

ReactNativeClient/lib/services/ResourceEditWatcher.ts

@@ -0,0 +1,229 @@
+const { Logger } = require('lib/logger.js');
+const Setting = require('lib/models/Setting');
+const Resource = require('lib/models/Resource');
+const { shim } = require('lib/shim');
+const EventEmitter = require('events');
+const chokidar = require('chokidar');
+const { bridge } = require('electron').remote.require('./bridge');
+const { _ } = require('lib/locale');
+import AsyncActionQueue from '../AsyncActionQueue';
+
+interface WatchedItem {
+	resourceId: string,
+	lastFileUpdatedTime: number,
+	lastResourceUpdatedTime: number,
+	path:string,
+	asyncSaveQueue: AsyncActionQueue,
+}
+
+interface WatchedItems {
+	[key:string]: WatchedItem,
+}
+
+export default class ResourceEditWatcher {
+
+	private static instance_:ResourceEditWatcher;
+
+	private logger_:any;
+	// private dispatch:Function;
+	private watcher_:any;
+	private chokidar_:any;
+	private watchedItems_:WatchedItems = {};
+	private eventEmitter_:any;
+	private tempDir_:string = '';
+
+	constructor() {
+		this.logger_ = new Logger();
+		// this.dispatch = () => {};
+		this.watcher_ = null;
+		this.chokidar_ = chokidar;
+		this.eventEmitter_ = new EventEmitter();
+	}
+
+	initialize(logger:any/* , dispatch:Function*/) {
+		this.logger_ = logger;
+		// this.dispatch = dispatch;
+	}
+
+	static instance() {
+		if (this.instance_) return this.instance_;
+		this.instance_ = new ResourceEditWatcher();
+		return this.instance_;
+	}
+
+	private async tempDir() {
+		if (!this.tempDir_) {
+			this.tempDir_ = `${Setting.value('tempDir')}/edited_resources`;
+			await shim.fsDriver().mkdir(this.tempDir_);
+		}
+
+		return this.tempDir_;
+	}
+
+	logger() {
+		return this.logger_;
+	}
+
+	on(eventName:string, callback:Function) {
+		return this.eventEmitter_.on(eventName, callback);
+	}
+
+	off(eventName:string, callback:Function) {
+		return this.eventEmitter_.removeListener(eventName, callback);
+	}
+
+	private watch(fileToWatch:string) {
+		if (!this.chokidar_) return;
+
+		const makeSaveAction = (resourceId:string, path:string) => {
+			return async () => {
+				this.logger().info(`ResourceEditWatcher: Saving resource ${resourceId}`);
+				const resource = await Resource.load(resourceId);
+				const watchedItem = this.watchedItemByResourceId(resourceId);
+
+				if (resource.updated_time !== watchedItem.lastResourceUpdatedTime) {
+					this.logger().info(`ResourceEditWatcher: Conflict was detected (resource was modified from somewhere else, possibly via sync). Conflict note will be created: ${resourceId}`);
+					// The resource has been modified from elsewhere, for example via sync
+					// so copy the current version to the Conflict notebook, and overwrite
+					// the resource content.
+					await Resource.createConflictResourceNote(resource);
+				}
+
+				const savedResource = await Resource.updateResourceBlobContent(resourceId, path);
+				watchedItem.lastResourceUpdatedTime = savedResource.updated_time;
+				this.eventEmitter_.emit('resourceChange', { id: resourceId });
+			};
+		};
+
+		if (!this.watcher_) {
+			this.watcher_ = this.chokidar_.watch(fileToWatch);
+			this.watcher_.on('all', async (event:any, path:string) => {
+				this.logger().info(`ResourceEditWatcher: Event: ${event}: ${path}`);
+
+				if (event === 'unlink') {
+					// File are unwatched in the stopWatching functions below. When we receive an unlink event
+					// here it might be that the file is quickly moved to a different location and replaced by
+					// another file with the same name, as it happens with emacs. So because of this
+					// we keep watching anyway.
+					// See: https://github.com/laurent22/joplin/issues/710#issuecomment-420997167
+					// this.watcher_.unwatch(path);
+				} else if (event === 'change') {
+					const watchedItem = this.watchedItemByPath(path);
+					const resourceId = watchedItem.resourceId;
+
+					if (!watchedItem) {
+						this.logger().error(`ResourceEditWatcher: could not find resource ID from path: ${path}`);
+						return;
+					}
+
+					const stat = await shim.fsDriver().stat(path);
+					const editedFileUpdatedTime = stat.mtime.getTime();
+
+					if (watchedItem.lastFileUpdatedTime === editedFileUpdatedTime) {
+						// chokidar is buggy and emits "change" events even when nothing has changed
+						// so double-check the modified time and skip processing if there's no change.
+						// In particular it emits two such events just after the file has been copied
+						// in openAndWatch().
+						this.logger().debug(`ResourceEditWatcher: No timestamp change - skip: ${resourceId}`);
+						return;
+					}
+
+					this.logger().debug(`ResourceEditWatcher: Queuing save action: ${resourceId}`);
+
+					watchedItem.asyncSaveQueue.push(makeSaveAction(resourceId, path));
+					watchedItem.lastFileUpdatedTime = editedFileUpdatedTime;
+				} else if (event === 'error') {
+					this.logger().error('ResourceEditWatcher: error');
+				}
+			});
+			// Hack to support external watcher on some linux applications (gedit, gvim, etc)
+			// taken from https://github.com/paulmillr/chokidar/issues/591
+			// @ts-ignore Leave unused path variable
+			this.watcher_.on('raw', async (event:string, path:string, options:any) => {
+				if (event === 'rename') {
+					this.watcher_.unwatch(options.watchedPath);
+					this.watcher_.add(options.watchedPath);
+				}
+			});
+		} else {
+			this.watcher_.add(fileToWatch);
+		}
+
+		return this.watcher_;
+	}
+
+	public async openAndWatch(resourceId:string) {
+		let watchedItem = this.watchedItemByResourceId(resourceId);
+
+		if (!watchedItem) {
+			// Immediately create and push the item to prevent race conditions
+
+			watchedItem = {
+				resourceId: resourceId,
+				lastFileUpdatedTime: 0,
+				lastResourceUpdatedTime: 0,
+				asyncSaveQueue: new AsyncActionQueue(1000),
+				path: '',
+			};
+
+			this.watchedItems_[resourceId] = watchedItem;
+
+			const resource = await Resource.load(resourceId);
+			if (!(await Resource.isReady(resource))) throw new Error(_('This attachment is not downloaded or not decrypted yet'));
+			const sourceFilePath = Resource.fullPath(resource);
+			const tempDir = await this.tempDir();
+			const editFilePath = await shim.fsDriver().findUniqueFilename(`${tempDir}/${Resource.friendlySafeFilename(resource)}`);
+			await shim.fsDriver().copy(sourceFilePath, editFilePath);
+			const stat = await shim.fsDriver().stat(editFilePath);
+
+			watchedItem.path = editFilePath;
+			watchedItem.lastFileUpdatedTime = stat.mtime.getTime();
+			watchedItem.lastResourceUpdatedTime = resource.updated_time;
+
+			this.watch(editFilePath);
+		}
+
+		bridge().openItem(watchedItem.path);
+
+		this.logger().info(`ResourceEditWatcher: Started watching ${watchedItem.path}`);
+	}
+
+	async stopWatching(resourceId:string) {
+		if (!resourceId) return;
+
+		const item = this.watchedItemByResourceId(resourceId);
+		if (!item) {
+			this.logger().error(`ResourceEditWatcher: Trying to stop watching non-watched resource ${resourceId}`);
+			return;
+		}
+
+		await item.asyncSaveQueue.waitForAllDone();
+
+		if (this.watcher_) this.watcher_.unwatch(item.path);
+		await shim.fsDriver().remove(item.path);
+		delete this.watchedItems_[resourceId];
+		this.logger().info(`ResourceEditWatcher: Stopped watching ${item.path}`);
+	}
+
+	public async stopWatchingAll() {
+		const promises = [];
+		for (const resourceId in this.watchedItems_) {
+			const item = this.watchedItems_[resourceId];
+			promises.push(this.stopWatching(item.resourceId));
+		}
+		return Promise.all(promises);
+	}
+
+	private watchedItemByResourceId(resourceId:string):WatchedItem {
+		return this.watchedItems_[resourceId];
+	}
+
+	private watchedItemByPath(path:string):WatchedItem {
+		for (const resourceId in this.watchedItems_) {
+			const item = this.watchedItems_[resourceId];
+			if (item.path === path) return item;
+		}
+		return null;
+	}
+
+}

ReactNativeClient/lib/shim-init-node.js

@@ -144,7 +144,7 @@ function shimInit() {
 
 	shim.createResourceFromPath = async function(filePath, defaultProps = null, options = null) {
 		options = Object.assign({
-			resizeLargeImages: 'always', // 'always' or 'ask'
+			resizeLargeImages: 'always', // 'always', 'ask' or 'never'
 		}, options);
 
 		const readChunk = require('read-chunk');
@@ -182,7 +182,7 @@ function shimInit() {
 
 		const targetPath = Resource.fullPath(resource);
 
-		if (['image/jpeg', 'image/jpg', 'image/png'].includes(resource.mime)) {
+		if (options.resizeLargeImages !== 'never' && ['image/jpeg', 'image/jpg', 'image/png'].includes(resource.mime)) {
 			const ok = await handleResizeImage_(filePath, targetPath, resource.mime, options.resizeLargeImages);
 			if (!ok) return null;
 		} else {

ReactNativeClient/lib/synchronizer.js

@@ -372,6 +372,12 @@ class Synchronizer {
 						let reason = '';
 						let remoteContent = null;
 
+						const getConflictType = (conflictedItem) => {
+							if (conflictedItem.type_ === BaseModel.TYPE_NOTE) return 'noteConflict';
+							if (conflictedItem.type_ === BaseModel.TYPE_RESOURCE) return 'resourceConflict';
+							return 'itemConflict';
+						};
+
 						if (!remote) {
 							if (!local.sync_time) {
 								action = 'createRemote';
@@ -379,7 +385,7 @@ class Synchronizer {
 							} else {
 								// Note or item was modified after having been deleted remotely
 								// "itemConflict" is for all the items except the notes, which are dealt with in a special way
-								action = local.type_ == BaseModel.TYPE_NOTE ? 'noteConflict' : 'itemConflict';
+								action = getConflictType(local);
 								reason = 'remote has been deleted, but local has changes';
 							}
 						} else {
@@ -416,7 +422,7 @@ class Synchronizer {
 								// Since, in this loop, we are only dealing with items that require sync, if the
 								// remote has been modified after the sync time, it means both items have been
 								// modified and so there's a conflict.
-								action = local.type_ == BaseModel.TYPE_NOTE ? 'noteConflict' : 'itemConflict';
+								action = getConflictType(local);
 								reason = 'both remote and local have changes';
 							} else {
 								action = 'updateRemote';
@@ -528,8 +534,25 @@ class Synchronizer {
 								conflictedNote.is_conflict = 1;
 								await Note.save(conflictedNote, { autoTimestamp: false, changeSource: ItemChange.SOURCE_SYNC });
 							}
-
+						} else if (action == 'resourceConflict') {
 							// ------------------------------------------------------------------------------
+							// Unlike notes we always handle the conflict for resources
+							// ------------------------------------------------------------------------------
+
+							await Resource.createConflictResourceNote(local);
+
+							if (remote) {
+								// The local content we have is no longer valid and should be re-downloaded
+								await Resource.setLocalState(local.id, {
+									fetch_status: Resource.FETCH_STATUS_IDLE,
+								});
+							}
+						}
+
+						if (['noteConflict', 'resourceConflict'].includes(action)) {
+							// ------------------------------------------------------------------------------
+							// For note and resource conflicts, the creation of the conflict item is done
+							// differently. However the way the local content is handled is the same.
 							// Either copy the remote content to local or, if the remote content has
 							// been deleted, delete the local content.
 							// ------------------------------------------------------------------------------
@@ -718,9 +741,9 @@ class Synchronizer {
 							if (action == 'createLocal') options.isNew = true;
 							if (action == 'updateLocal') options.oldItem = local;
 
-							const creatingNewResource = content.type_ == BaseModel.TYPE_RESOURCE && action == 'createLocal';
+							const creatingOrUpdatingResource = content.type_ == BaseModel.TYPE_RESOURCE && (action == 'createLocal' || action == 'updateLocal');
 
-							if (creatingNewResource) {
+							if (creatingOrUpdatingResource) {
 								if (content.size >= this.maxResourceSize()) {
 									await handleCannotSyncItem(ItemClass, syncTargetId, content, `File "${content.title}" is larger than allowed ${this.maxResourceSize()} bytes. Beyond this limit, the mobile app would crash.`, BaseItem.SYNC_ITEM_LOCATION_REMOTE);
 									continue;
@@ -731,7 +754,7 @@ class Synchronizer {
 
 							await ItemClass.save(content, options);
 
-							if (creatingNewResource) this.dispatch({ type: 'SYNC_CREATED_RESOURCE', id: content.id });
+							if (creatingOrUpdatingResource) this.dispatch({ type: 'SYNC_CREATED_OR_UPDATED_RESOURCE', id: content.id });
 
 							if (!hasAutoEnabledEncryption && content.type_ === BaseModel.TYPE_MASTER_KEY && !masterKeysBefore) {
 								hasAutoEnabledEncryption = true;

ReactNativeClient/root.js

@@ -151,7 +151,7 @@ const generalMiddleware = store => next => async (action) => {
 		DecryptionWorker.instance().scheduleStart();
 	}
 
-	if (action.type === 'SYNC_CREATED_RESOURCE') {
+	if (action.type === 'SYNC_CREATED_OR_UPDATED_RESOURCE') {
 		ResourceFetcher.instance().autoAddResources();
 	}
 

joplin.sublime-project

@@ -70,7 +70,10 @@
 				"ElectronClient/gui/NoteEditor/utils/useResourceRefresher.js",
 				"ElectronClient/gui/NoteEditor/utils/useFormNote.js",
 				"ElectronClient/gui/NoteEditor/utils/useResourceInfos.js",
-				"Modules/TinyMCE/langs/*.js"
+				"Modules/TinyMCE/langs/*.js",
+				"ElectronClient/gui/NoteEditor/utils/contextMenu.js",
+				"ReactNativeClient/lib/services/ResourceEditWatcher.js",
+				"ReactNativeClient/lib/hooks/usePrevious.js"
 			],
 			"folder_exclude_patterns":
 			[

readme/conflict.md

@@ -0,0 +1,17 @@
+# What is a conflict?
+
+A conflict happens when one note or one attachment is modified in two different places, and then synchronised. In that case, it not possible to determine which version of the note or attachment you want to keep, and thus a conflict is generated.
+
+# What happens in case of a conflict?
+
+When Joplin detects a conflict, the local note is copied to the Conflict notebook so as to avoid any data loss. Then the remote note is downloaded. You can then inspect the notes in the Conflict notebook, compare it with your other version, and copy any change that might have been overwritten.
+
+# How to avoid conflicts?
+
+Conflicts are always annoying to deal with so it is best to avoid them as much as possible.
+
+For this, the best way is to synchronise as often as possible, so that you are always working with the latest versions of your notes.
+
+Joplin attempts to do this by uploading your latest changes within a few seconds. However, downloading changes is done at fixed intervals, every few minutes (as defined in the Config screen) and this is where conflicts may happen. It can also happen if one of your device did not have an internet connection for some times, and then synchronises. A bad internet connection can also hinder synchronisation because it will interrupt the process, which may have to restarted from the beginning to ensure consistency.
+
+So if you have not opened your application in a while, manually sync it and wait for it to complete, that way you are sure that whatever change you make will be on the latest version of the note.