From 65f63e273bc37c742e44fa29d375f95386a0a577 Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Tue, 1 Dec 2020 17:40:20 +0000 Subject: [PATCH] splitting sync tests --- .eslintignore | 6 + .gitignore | 6 + packages/app-cli/tests/Synchronizer.basics.ts | 137 ++- packages/app-cli/tests/Synchronizer.e2ee.ts | 409 +++++++++ .../app-cli/tests/Synchronizer.resources.ts | 362 ++++++++ packages/app-cli/tests/Synchronizer.ts | 836 ------------------ 6 files changed, 919 insertions(+), 837 deletions(-) create mode 100644 packages/app-cli/tests/Synchronizer.e2ee.ts create mode 100644 packages/app-cli/tests/Synchronizer.resources.ts diff --git a/.eslintignore b/.eslintignore index 5f6fdd3149..fa4f3c786d 100644 --- a/.eslintignore +++ b/.eslintignore @@ -81,8 +81,14 @@ packages/app-cli/tests/Synchronizer.conflicts.d.ts packages/app-cli/tests/Synchronizer.conflicts.js packages/app-cli/tests/Synchronizer.conflicts.js.map packages/app-cli/tests/Synchronizer.d.ts +packages/app-cli/tests/Synchronizer.e2ee.d.ts +packages/app-cli/tests/Synchronizer.e2ee.js +packages/app-cli/tests/Synchronizer.e2ee.js.map packages/app-cli/tests/Synchronizer.js packages/app-cli/tests/Synchronizer.js.map +packages/app-cli/tests/Synchronizer.resources.d.ts +packages/app-cli/tests/Synchronizer.resources.js +packages/app-cli/tests/Synchronizer.resources.js.map packages/app-cli/tests/fsDriver.d.ts packages/app-cli/tests/fsDriver.js packages/app-cli/tests/fsDriver.js.map diff --git a/.gitignore b/.gitignore index 9b4cb1c95e..d2f1c5759a 100644 --- a/.gitignore +++ b/.gitignore @@ -72,8 +72,14 @@ packages/app-cli/tests/Synchronizer.conflicts.d.ts packages/app-cli/tests/Synchronizer.conflicts.js packages/app-cli/tests/Synchronizer.conflicts.js.map packages/app-cli/tests/Synchronizer.d.ts +packages/app-cli/tests/Synchronizer.e2ee.d.ts +packages/app-cli/tests/Synchronizer.e2ee.js +packages/app-cli/tests/Synchronizer.e2ee.js.map packages/app-cli/tests/Synchronizer.js packages/app-cli/tests/Synchronizer.js.map +packages/app-cli/tests/Synchronizer.resources.d.ts +packages/app-cli/tests/Synchronizer.resources.js +packages/app-cli/tests/Synchronizer.resources.js.map packages/app-cli/tests/fsDriver.d.ts packages/app-cli/tests/fsDriver.js packages/app-cli/tests/fsDriver.js.map diff --git a/packages/app-cli/tests/Synchronizer.basics.ts b/packages/app-cli/tests/Synchronizer.basics.ts index 1342b912d0..68f8635197 100644 --- a/packages/app-cli/tests/Synchronizer.basics.ts +++ b/packages/app-cli/tests/Synchronizer.basics.ts @@ -1,9 +1,11 @@ +import Setting from '@joplin/lib/models/Setting'; import { allNotesFolders, remoteNotesAndFolders, localNotesFoldersSameAsRemote } from './test-utils-synchronizer'; -const { synchronizerStart, setupDatabaseAndSynchronizer, synchronizer, sleep, switchClient, syncTargetId } = require('./test-utils.js'); +const { syncTargetName, synchronizerStart, setupDatabaseAndSynchronizer, synchronizer, sleep, switchClient, syncTargetId, fileApi } = require('./test-utils.js'); const Folder = require('@joplin/lib/models/Folder.js'); const Note = require('@joplin/lib/models/Note.js'); const BaseItem = require('@joplin/lib/models/BaseItem.js'); +const WelcomeUtils = require('@joplin/lib/WelcomeUtils'); let insideBeforeEach = false; @@ -266,4 +268,137 @@ describe('Synchronizer.basics', function() { expect(disabledItems.length).toBe(1); })); + it('should allow duplicate folder titles', (async () => { + await Folder.save({ title: 'folder' }); + + await switchClient(2); + + let remoteF2 = await Folder.save({ title: 'folder' }); + await synchronizerStart(); + + await switchClient(1); + + await sleep(0.1); + + await synchronizerStart(); + + const localF2 = await Folder.load(remoteF2.id); + + expect(localF2.title == remoteF2.title).toBe(true); + + // Then that folder that has been renamed locally should be set in such a way + // that synchronizing it applies the title change remotely, and that new title + // should be retrieved by client 2. + + await synchronizerStart(); + + await switchClient(2); + await sleep(0.1); + + await synchronizerStart(); + + remoteF2 = await Folder.load(remoteF2.id); + + expect(remoteF2.title == localF2.title).toBe(true); + })); + + it('should create remote items with UTF-8 content', (async () => { + const folder = await Folder.save({ title: 'Fahrräder' }); + await Note.save({ title: 'Fahrräder', body: 'Fahrräder', parent_id: folder.id }); + const all = await allNotesFolders(); + + await synchronizerStart(); + + await localNotesFoldersSameAsRemote(all, expect); + })); + + it('should update remote items but not pull remote changes', (async () => { + const folder = await Folder.save({ title: 'folder1' }); + const note = await Note.save({ title: 'un', parent_id: folder.id }); + await synchronizerStart(); + + await switchClient(2); + + await synchronizerStart(); + await Note.save({ title: 'deux', parent_id: folder.id }); + await synchronizerStart(); + + await switchClient(1); + + await Note.save({ title: 'un UPDATE', id: note.id }); + await synchronizerStart(null, { syncSteps: ['update_remote'] }); + const all = await allNotesFolders(); + expect(all.length).toBe(2); + + await switchClient(2); + + await synchronizerStart(); + const note2 = await Note.load(note.id); + expect(note2.title).toBe('un UPDATE'); + })); + + it('should create a new Welcome notebook on each client', (async () => { + // Create the Welcome items on two separate clients + + await WelcomeUtils.createWelcomeItems(); + await synchronizerStart(); + + await switchClient(2); + + await WelcomeUtils.createWelcomeItems(); + const beforeFolderCount = (await Folder.all()).length; + const beforeNoteCount = (await Note.all()).length; + expect(beforeFolderCount === 1).toBe(true); + expect(beforeNoteCount > 1).toBe(true); + + await synchronizerStart(); + + const afterFolderCount = (await Folder.all()).length; + const afterNoteCount = (await Note.all()).length; + + expect(afterFolderCount).toBe(beforeFolderCount * 2); + expect(afterNoteCount).toBe(beforeNoteCount * 2); + + // Changes to the Welcome items should be synced to all clients + + const f1 = (await Folder.all())[0]; + await Folder.save({ id: f1.id, title: 'Welcome MOD' }); + + await synchronizerStart(); + + await switchClient(1); + + await synchronizerStart(); + + const f1_1 = await Folder.load(f1.id); + expect(f1_1.title).toBe('Welcome MOD'); + })); + + it('should not wipe out user data when syncing with an empty target', (async () => { + // Only these targets support the wipeOutFailSafe flag (in other words, the targets that use basicDelta) + if (!['nextcloud', 'memory', 'filesystem', 'amazon_s3'].includes(syncTargetName())) return; + + for (let i = 0; i < 10; i++) await Note.save({ title: 'note' }); + + Setting.setValue('sync.wipeOutFailSafe', true); + await synchronizerStart(); + await fileApi().clearRoot(); // oops + await synchronizerStart(); + expect((await Note.all()).length).toBe(10); // but since the fail-safe if on, the notes have not been deleted + + Setting.setValue('sync.wipeOutFailSafe', false); // Now switch it off + await synchronizerStart(); + expect((await Note.all()).length).toBe(0); // Since the fail-safe was off, the data has been cleared + + // Handle case where the sync target has been wiped out, then the user creates one note and sync. + + for (let i = 0; i < 10; i++) await Note.save({ title: 'note' }); + Setting.setValue('sync.wipeOutFailSafe', true); + await synchronizerStart(); + await fileApi().clearRoot(); + await Note.save({ title: 'ma note encore' }); + await synchronizerStart(); + expect((await Note.all()).length).toBe(11); + })); + }); \ No newline at end of file diff --git a/packages/app-cli/tests/Synchronizer.e2ee.ts b/packages/app-cli/tests/Synchronizer.e2ee.ts new file mode 100644 index 0000000000..5030a3fd70 --- /dev/null +++ b/packages/app-cli/tests/Synchronizer.e2ee.ts @@ -0,0 +1,409 @@ +import time from '@joplin/lib/time'; +import shim from '@joplin/lib/shim'; +import Setting from '@joplin/lib/models/Setting'; +import BaseModel from '@joplin/lib/BaseModel'; +import { NoteEntity } from '@joplin/lib/services/database/types'; +import { allNotesFolders, remoteNotesAndFolders, remoteNotesFoldersResources, remoteResources, localNotesFoldersSameAsRemote } from './test-utils-synchronizer'; + +const { synchronizerStart, syncTargetName, allSyncTargetItemsEncrypted, tempFilePath, resourceFetcher, kvStore, revisionService, setupDatabaseAndSynchronizer, synchronizer, fileApi, sleep, switchClient, syncTargetId, encryptionService, loadEncryptionMasterKey, fileContentEqual, decryptionWorker, checkThrowAsync } = require('./test-utils.js'); +const Folder = require('@joplin/lib/models/Folder.js'); +const Note = require('@joplin/lib/models/Note.js'); +const Resource = require('@joplin/lib/models/Resource.js'); +const ResourceFetcher = require('@joplin/lib/services/ResourceFetcher'); +const Tag = require('@joplin/lib/models/Tag.js'); +const MasterKey = require('@joplin/lib/models/MasterKey'); +const BaseItem = require('@joplin/lib/models/BaseItem.js'); +const Revision = require('@joplin/lib/models/Revision.js'); +const WelcomeUtils = require('@joplin/lib/WelcomeUtils'); + +let insideBeforeEach = false; + +describe('synchronizer', function() { + + beforeEach(async (done) => { + insideBeforeEach = true; + + await setupDatabaseAndSynchronizer(1); + await setupDatabaseAndSynchronizer(2); + await switchClient(1); + done(); + + insideBeforeEach = false; + }); + + it('notes and folders should get encrypted when encryption is enabled', (async () => { + Setting.setValue('encryption.enabled', true); + const masterKey = await loadEncryptionMasterKey(); + const folder1 = await Folder.save({ title: 'folder1' }); + let note1 = await Note.save({ title: 'un', body: 'to be encrypted', parent_id: folder1.id }); + await synchronizerStart(); + // After synchronisation, remote items should be encrypted but local ones remain plain text + note1 = await Note.load(note1.id); + expect(note1.title).toBe('un'); + + await switchClient(2); + + await synchronizerStart(); + let folder1_2 = await Folder.load(folder1.id); + let note1_2 = await Note.load(note1.id); + const masterKey_2 = await MasterKey.load(masterKey.id); + // On this side however it should be received encrypted + expect(!note1_2.title).toBe(true); + expect(!folder1_2.title).toBe(true); + expect(!!note1_2.encryption_cipher_text).toBe(true); + expect(!!folder1_2.encryption_cipher_text).toBe(true); + // Master key is already encrypted so it does not get re-encrypted during sync + expect(masterKey_2.content).toBe(masterKey.content); + expect(masterKey_2.checksum).toBe(masterKey.checksum); + // Now load the master key we got from client 1 and try to decrypt + await encryptionService().loadMasterKey_(masterKey_2, '123456', true); + // Get the decrypted items back + await Folder.decrypt(folder1_2); + await Note.decrypt(note1_2); + folder1_2 = await Folder.load(folder1.id); + note1_2 = await Note.load(note1.id); + // Check that properties match the original items. Also check + // the encryption did not affect the updated_time timestamp. + expect(note1_2.title).toBe(note1.title); + expect(note1_2.body).toBe(note1.body); + expect(note1_2.updated_time).toBe(note1.updated_time); + expect(!note1_2.encryption_cipher_text).toBe(true); + expect(folder1_2.title).toBe(folder1.title); + expect(folder1_2.updated_time).toBe(folder1.updated_time); + expect(!folder1_2.encryption_cipher_text).toBe(true); + })); + + it('should enable encryption automatically when downloading new master key (and none was previously available)',(async () => { + // Enable encryption on client 1 and sync an item + Setting.setValue('encryption.enabled', true); + await loadEncryptionMasterKey(); + let folder1 = await Folder.save({ title: 'folder1' }); + await synchronizerStart(); + + await switchClient(2); + + // Synchronising should enable encryption since we're going to get a master key + expect(Setting.value('encryption.enabled')).toBe(false); + await synchronizerStart(); + expect(Setting.value('encryption.enabled')).toBe(true); + + // Check that we got the master key from client 1 + const masterKey = (await MasterKey.all())[0]; + expect(!!masterKey).toBe(true); + + // Since client 2 hasn't supplied a password yet, no master key is currently loaded + expect(encryptionService().loadedMasterKeyIds().length).toBe(0); + + // If we sync now, nothing should be sent to target since we don't have a password. + // Technically it's incorrect to set the property of an encrypted variable but it allows confirming + // that encryption doesn't work if user hasn't supplied a password. + await BaseItem.forceSync(folder1.id); + await synchronizerStart(); + + await switchClient(1); + + await synchronizerStart(); + folder1 = await Folder.load(folder1.id); + expect(folder1.title).toBe('folder1'); // Still at old value + + await switchClient(2); + + // Now client 2 set the master key password + Setting.setObjectValue('encryption.passwordCache', masterKey.id, '123456'); + await encryptionService().loadMasterKeysFromSettings(); + + // Now that master key should be loaded + expect(encryptionService().loadedMasterKeyIds()[0]).toBe(masterKey.id); + + // Decrypt all the data. Now change the title and sync again - this time the changes should be transmitted + await decryptionWorker().start(); + await Folder.save({ id: folder1.id, title: 'change test' }); + + // If we sync now, this time client 1 should get the changes we did earlier + await synchronizerStart(); + + await switchClient(1); + + await synchronizerStart(); + // Decrypt the data we just got + await decryptionWorker().start(); + folder1 = await Folder.load(folder1.id); + expect(folder1.title).toBe('change test'); // Got title from client 2 + })); + + it('should encrypt existing notes too when enabling E2EE', (async () => { + // First create a folder, without encryption enabled, and sync it + await Folder.save({ title: 'folder1' }); + await synchronizerStart(); + let files = await fileApi().list('', { includeDirs: false, syncItemsOnly: true }); + let content = await fileApi().get(files.items[0].path); + expect(content.indexOf('folder1') >= 0).toBe(true); + + // Then enable encryption and sync again + let masterKey = await encryptionService().generateMasterKey('123456'); + masterKey = await MasterKey.save(masterKey); + await encryptionService().enableEncryption(masterKey, '123456'); + await encryptionService().loadMasterKeysFromSettings(); + await synchronizerStart(); + + // Even though the folder has not been changed it should have been synced again so that + // an encrypted version of it replaces the decrypted version. + files = await fileApi().list('', { includeDirs: false, syncItemsOnly: true }); + expect(files.items.length).toBe(2); + // By checking that the folder title is not present, we can confirm that the item has indeed been encrypted + // One of the two items is the master key + content = await fileApi().get(files.items[0].path); + expect(content.indexOf('folder1') < 0).toBe(true); + content = await fileApi().get(files.items[1].path); + expect(content.indexOf('folder1') < 0).toBe(true); + })); + + it('should upload decrypted items to sync target after encryption disabled', (async () => { + Setting.setValue('encryption.enabled', true); + await loadEncryptionMasterKey(); + + await Folder.save({ title: 'folder1' }); + await synchronizerStart(); + + let allEncrypted = await allSyncTargetItemsEncrypted(); + expect(allEncrypted).toBe(true); + + await encryptionService().disableEncryption(); + + await synchronizerStart(); + allEncrypted = await allSyncTargetItemsEncrypted(); + expect(allEncrypted).toBe(false); + })); + + it('should not upload any item if encryption was enabled, and items have not been decrypted, and then encryption disabled', (async () => { + // For some reason I can't explain, this test is sometimes executed before beforeEach is finished + // which means it's going to fail in unexpected way. So the loop below wait for beforeEach to be done. + while (insideBeforeEach) await time.msleep(100); + + Setting.setValue('encryption.enabled', true); + const masterKey = await loadEncryptionMasterKey(); + + await Folder.save({ title: 'folder1' }); + await synchronizerStart(); + + await switchClient(2); + + await synchronizerStart(); + expect(Setting.value('encryption.enabled')).toBe(true); + + // If we try to disable encryption now, it should throw an error because some items are + // currently encrypted. They must be decrypted first so that they can be sent as + // plain text to the sync target. + // let hasThrown = await checkThrowAsync(async () => await encryptionService().disableEncryption()); + // expect(hasThrown).toBe(true); + + // Now supply the password, and decrypt the items + Setting.setObjectValue('encryption.passwordCache', masterKey.id, '123456'); + await encryptionService().loadMasterKeysFromSettings(); + await decryptionWorker().start(); + + // Try to disable encryption again + const hasThrown = await checkThrowAsync(async () => await encryptionService().disableEncryption()); + expect(hasThrown).toBe(false); + + // If we sync now the target should receive the decrypted items + await synchronizerStart(); + const allEncrypted = await allSyncTargetItemsEncrypted(); + expect(allEncrypted).toBe(false); + })); + + it('should set the resource file size after decryption', (async () => { + Setting.setValue('encryption.enabled', true); + const masterKey = await loadEncryptionMasterKey(); + + const folder1 = await Folder.save({ title: 'folder1' }); + const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id }); + await shim.attachFileToNote(note1, `${__dirname}/../tests/support/photo.jpg`); + const resource1 = (await Resource.all())[0]; + await Resource.setFileSizeOnly(resource1.id, -1); + Resource.fullPath(resource1); + await synchronizerStart(); + + await switchClient(2); + + await synchronizerStart(); + Setting.setObjectValue('encryption.passwordCache', masterKey.id, '123456'); + await encryptionService().loadMasterKeysFromSettings(); + + const fetcher = new ResourceFetcher(() => { return synchronizer().api(); }); + fetcher.queueDownload_(resource1.id); + await fetcher.waitForAllFinished(); + await decryptionWorker().start(); + + const resource1_2 = await Resource.load(resource1.id); + expect(resource1_2.size).toBe(2720); + })); + + it('should encrypt remote resources after encryption has been enabled', (async () => { + while (insideBeforeEach) await time.msleep(100); + + const folder1 = await Folder.save({ title: 'folder1' }); + const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id }); + await shim.attachFileToNote(note1, `${__dirname}/../tests/support/photo.jpg`); + await synchronizerStart(); + + expect(await allSyncTargetItemsEncrypted()).toBe(false); + + const masterKey = await loadEncryptionMasterKey(); + await encryptionService().enableEncryption(masterKey, '123456'); + await encryptionService().loadMasterKeysFromSettings(); + + await synchronizerStart(); + + expect(await allSyncTargetItemsEncrypted()).toBe(true); + })); + + it('should upload encrypted resource, but it should not mark the blob as encrypted locally', (async () => { + while (insideBeforeEach) await time.msleep(100); + + const folder1 = await Folder.save({ title: 'folder1' }); + const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id }); + await shim.attachFileToNote(note1, `${__dirname}/../tests/support/photo.jpg`); + const masterKey = await loadEncryptionMasterKey(); + await encryptionService().enableEncryption(masterKey, '123456'); + await encryptionService().loadMasterKeysFromSettings(); + await synchronizerStart(); + + const resource1 = (await Resource.all())[0]; + expect(resource1.encryption_blob_encrypted).toBe(0); + })); + + it('should decrypt the resource metadata, but not try to decrypt the file, if it is not present', (async () => { + const note1 = await Note.save({ title: 'note' }); + await shim.attachFileToNote(note1, `${__dirname}/../tests/support/photo.jpg`); + const masterKey = await loadEncryptionMasterKey(); + await encryptionService().enableEncryption(masterKey, '123456'); + await encryptionService().loadMasterKeysFromSettings(); + await synchronizerStart(); + expect(await allSyncTargetItemsEncrypted()).toBe(true); + + await switchClient(2); + + await synchronizerStart(); + Setting.setObjectValue('encryption.passwordCache', masterKey.id, '123456'); + await encryptionService().loadMasterKeysFromSettings(); + await decryptionWorker().start(); + + let resource = (await Resource.all())[0]; + + expect(!!resource.encryption_applied).toBe(false); + expect(!!resource.encryption_blob_encrypted).toBe(true); + + const resourceFetcher = new ResourceFetcher(() => { return synchronizer().api(); }); + await resourceFetcher.start(); + await resourceFetcher.waitForAllFinished(); + + const ls = await Resource.localState(resource); + expect(ls.fetch_status).toBe(Resource.FETCH_STATUS_DONE); + + await decryptionWorker().start(); + resource = (await Resource.all())[0]; + + expect(!!resource.encryption_blob_encrypted).toBe(false); + })); + + it('should stop trying to decrypt item after a few attempts', (async () => { + let hasThrown; + + const note = await Note.save({ title: 'ma note' }); + const masterKey = await loadEncryptionMasterKey(); + await encryptionService().enableEncryption(masterKey, '123456'); + await encryptionService().loadMasterKeysFromSettings(); + await synchronizerStart(); + + await switchClient(2); + + await synchronizerStart(); + + // First, simulate a broken note and check that the decryption worker + // gives up decrypting after a number of tries. This is mainly relevant + // for data that crashes the mobile application - we don't want to keep + // decrypting these. + + const encryptedNote = await Note.load(note.id); + const goodCipherText = encryptedNote.encryption_cipher_text; + await Note.save({ id: note.id, encryption_cipher_text: 'doesntlookright' }); + + Setting.setObjectValue('encryption.passwordCache', masterKey.id, '123456'); + await encryptionService().loadMasterKeysFromSettings(); + + hasThrown = await checkThrowAsync(async () => await decryptionWorker().start({ errorHandler: 'throw' })); + expect(hasThrown).toBe(true); + + hasThrown = await checkThrowAsync(async () => await decryptionWorker().start({ errorHandler: 'throw' })); + expect(hasThrown).toBe(true); + + // Third time, an error is logged and no error is thrown + hasThrown = await checkThrowAsync(async () => await decryptionWorker().start({ errorHandler: 'throw' })); + expect(hasThrown).toBe(false); + + const disabledItems = await decryptionWorker().decryptionDisabledItems(); + expect(disabledItems.length).toBe(1); + expect(disabledItems[0].id).toBe(note.id); + + expect((await kvStore().all()).length).toBe(1); + await kvStore().clear(); + + // Now check that if it fails once but succeed the second time, the note + // is correctly decrypted and the counters are cleared. + + hasThrown = await checkThrowAsync(async () => await decryptionWorker().start({ errorHandler: 'throw' })); + expect(hasThrown).toBe(true); + + await Note.save({ id: note.id, encryption_cipher_text: goodCipherText }); + + hasThrown = await checkThrowAsync(async () => await decryptionWorker().start({ errorHandler: 'throw' })); + expect(hasThrown).toBe(false); + + const decryptedNote = await Note.load(note.id); + expect(decryptedNote.title).toBe('ma note'); + + expect((await kvStore().all()).length).toBe(0); + expect((await decryptionWorker().decryptionDisabledItems()).length).toBe(0); + })); + + it('should not encrypt notes that are shared', (async () => { + Setting.setValue('encryption.enabled', true); + await loadEncryptionMasterKey(); + + const folder1 = await Folder.save({ title: 'folder1' }); + const note1 = await Note.save({ title: 'un', parent_id: folder1.id }); + let note2 = await Note.save({ title: 'deux', parent_id: folder1.id }); + await synchronizerStart(); + + await switchClient(2); + + await synchronizerStart(); + + await switchClient(1); + + const origNote2 = Object.assign({}, note2); + await BaseItem.updateShareStatus(note2, true); + note2 = await Note.load(note2.id); + + // Sharing a note should not modify the timestamps + expect(note2.user_updated_time).toBe(origNote2.user_updated_time); + expect(note2.user_created_time).toBe(origNote2.user_created_time); + + await synchronizerStart(); + + await switchClient(2); + + await synchronizerStart(); + + // The shared note should be decrypted + const note2_2 = await Note.load(note2.id); + expect(note2_2.title).toBe('deux'); + expect(note2_2.is_shared).toBe(1); + + // The non-shared note should be encrypted + const note1_2 = await Note.load(note1.id); + expect(note1_2.title).toBe(''); + })); + +}); \ No newline at end of file diff --git a/packages/app-cli/tests/Synchronizer.resources.ts b/packages/app-cli/tests/Synchronizer.resources.ts new file mode 100644 index 0000000000..c969505c97 --- /dev/null +++ b/packages/app-cli/tests/Synchronizer.resources.ts @@ -0,0 +1,362 @@ +import time from '@joplin/lib/time'; +import shim from '@joplin/lib/shim'; +import Setting from '@joplin/lib/models/Setting'; +import BaseModel from '@joplin/lib/BaseModel'; +import { NoteEntity } from '@joplin/lib/services/database/types'; +import { allNotesFolders, remoteNotesAndFolders, remoteNotesFoldersResources, remoteResources, localNotesFoldersSameAsRemote } from './test-utils-synchronizer'; + +const { synchronizerStart, syncTargetName, allSyncTargetItemsEncrypted, tempFilePath, resourceFetcher, kvStore, revisionService, setupDatabaseAndSynchronizer, synchronizer, fileApi, sleep, switchClient, syncTargetId, encryptionService, loadEncryptionMasterKey, fileContentEqual, decryptionWorker, checkThrowAsync } = require('./test-utils.js'); +const Folder = require('@joplin/lib/models/Folder.js'); +const Note = require('@joplin/lib/models/Note.js'); +const Resource = require('@joplin/lib/models/Resource.js'); +const ResourceFetcher = require('@joplin/lib/services/ResourceFetcher'); +const Tag = require('@joplin/lib/models/Tag.js'); +const MasterKey = require('@joplin/lib/models/MasterKey'); +const BaseItem = require('@joplin/lib/models/BaseItem.js'); +const Revision = require('@joplin/lib/models/Revision.js'); +const WelcomeUtils = require('@joplin/lib/WelcomeUtils'); + +let insideBeforeEach = false; + +describe('Synchronizer.conflicts', function() { + + beforeEach(async (done) => { + insideBeforeEach = true; + + await setupDatabaseAndSynchronizer(1); + await setupDatabaseAndSynchronizer(2); + await switchClient(1); + done(); + + insideBeforeEach = false; + }); + + it('should sync resources', (async () => { + while (insideBeforeEach) await time.msleep(500); + + const folder1 = await Folder.save({ title: 'folder1' }); + const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id }); + await shim.attachFileToNote(note1, `${__dirname}/../tests/support/photo.jpg`); + const resource1 = (await Resource.all())[0]; + const resourcePath1 = Resource.fullPath(resource1); + await synchronizerStart(); + expect((await remoteNotesFoldersResources()).length).toBe(3); + + await switchClient(2); + + await synchronizerStart(); + const allResources = await Resource.all(); + expect(allResources.length).toBe(1); + let resource1_2 = allResources[0]; + let ls = await Resource.localState(resource1_2); + expect(resource1_2.id).toBe(resource1.id); + expect(ls.fetch_status).toBe(Resource.FETCH_STATUS_IDLE); + + const fetcher = new ResourceFetcher(() => { return synchronizer().api(); }); + fetcher.queueDownload_(resource1_2.id); + await fetcher.waitForAllFinished(); + + resource1_2 = await Resource.load(resource1.id); + ls = await Resource.localState(resource1_2); + expect(ls.fetch_status).toBe(Resource.FETCH_STATUS_DONE); + + const resourcePath1_2 = Resource.fullPath(resource1_2); + expect(fileContentEqual(resourcePath1, resourcePath1_2)).toBe(true); + })); + + it('should handle resource download errors', (async () => { + while (insideBeforeEach) await time.msleep(500); + + const folder1 = await Folder.save({ title: 'folder1' }); + const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id }); + await shim.attachFileToNote(note1, `${__dirname}/../tests/support/photo.jpg`); + let resource1 = (await Resource.all())[0]; + await synchronizerStart(); + + await switchClient(2); + + await synchronizerStart(); + + const fetcher = new ResourceFetcher(() => { + return { + // Simulate a failed download + get: () => { return new Promise((_resolve: Function, reject: Function) => { reject(new Error('did not work')); }); }, + }; + }); + fetcher.queueDownload_(resource1.id); + await fetcher.waitForAllFinished(); + + resource1 = await Resource.load(resource1.id); + const ls = await Resource.localState(resource1); + expect(ls.fetch_status).toBe(Resource.FETCH_STATUS_ERROR); + expect(ls.fetch_error).toBe('did not work'); + })); + + it('should set the resource file size if it is missing', (async () => { + while (insideBeforeEach) await time.msleep(500); + + const folder1 = await Folder.save({ title: 'folder1' }); + const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id }); + await shim.attachFileToNote(note1, `${__dirname}/../tests/support/photo.jpg`); + await synchronizerStart(); + + await switchClient(2); + + await synchronizerStart(); + let r1 = (await Resource.all())[0]; + await Resource.setFileSizeOnly(r1.id, -1); + r1 = await Resource.load(r1.id); + expect(r1.size).toBe(-1); + + const fetcher = new ResourceFetcher(() => { return synchronizer().api(); }); + fetcher.queueDownload_(r1.id); + await fetcher.waitForAllFinished(); + r1 = await Resource.load(r1.id); + expect(r1.size).toBe(2720); + })); + + it('should delete resources', (async () => { + while (insideBeforeEach) await time.msleep(500); + + const folder1 = await Folder.save({ title: 'folder1' }); + const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id }); + await shim.attachFileToNote(note1, `${__dirname}/../tests/support/photo.jpg`); + const resource1 = (await Resource.all())[0]; + const resourcePath1 = Resource.fullPath(resource1); + await synchronizerStart(); + + await switchClient(2); + + await synchronizerStart(); + let allResources = await Resource.all(); + expect(allResources.length).toBe(1); + expect((await remoteNotesFoldersResources()).length).toBe(3); + await Resource.delete(resource1.id); + await synchronizerStart(); + expect((await remoteNotesFoldersResources()).length).toBe(2); + + const remoteBlob = await fileApi().stat(`.resource/${resource1.id}`); + expect(!remoteBlob).toBe(true); + + await switchClient(1); + + expect(await shim.fsDriver().exists(resourcePath1)).toBe(true); + await synchronizerStart(); + allResources = await Resource.all(); + expect(allResources.length).toBe(0); + expect(await shim.fsDriver().exists(resourcePath1)).toBe(false); + })); + + it('should encrypt resources', (async () => { + Setting.setValue('encryption.enabled', true); + const masterKey = await loadEncryptionMasterKey(); + + const folder1 = await Folder.save({ title: 'folder1' }); + const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id }); + await shim.attachFileToNote(note1, `${__dirname}/../tests/support/photo.jpg`); + const resource1 = (await Resource.all())[0]; + const resourcePath1 = Resource.fullPath(resource1); + await synchronizerStart(); + + await switchClient(2); + + await synchronizerStart(); + Setting.setObjectValue('encryption.passwordCache', masterKey.id, '123456'); + await encryptionService().loadMasterKeysFromSettings(); + + const fetcher = new ResourceFetcher(() => { return synchronizer().api(); }); + fetcher.queueDownload_(resource1.id); + await fetcher.waitForAllFinished(); + + let resource1_2 = (await Resource.all())[0]; + resource1_2 = await Resource.decrypt(resource1_2); + const resourcePath1_2 = Resource.fullPath(resource1_2); + + expect(fileContentEqual(resourcePath1, resourcePath1_2)).toBe(true); + })); + + it('should sync resource blob changes', (async () => { + const tempFile = tempFilePath('txt'); + await shim.fsDriver().writeFile(tempFile, '1234', 'utf8'); + const folder1 = await Folder.save({ title: 'folder1' }); + const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id }); + await shim.attachFileToNote(note1, tempFile); + await synchronizerStart(); + + await switchClient(2); + + await synchronizerStart(); + await resourceFetcher().start(); + await resourceFetcher().waitForAllFinished(); + let resource1_2 = (await Resource.all())[0]; + const modFile = tempFilePath('txt'); + await shim.fsDriver().writeFile(modFile, '1234 MOD', 'utf8'); + await Resource.updateResourceBlobContent(resource1_2.id, modFile); + const originalSize = resource1_2.size; + resource1_2 = (await Resource.all())[0]; + const newSize = resource1_2.size; + expect(originalSize).toBe(4); + expect(newSize).toBe(8); + await synchronizerStart(); + + await switchClient(1); + + await synchronizerStart(); + await resourceFetcher().start(); + await resourceFetcher().waitForAllFinished(); + const resource1_1 = (await Resource.all())[0]; + expect(resource1_1.size).toBe(newSize); + expect(await Resource.resourceBlobContent(resource1_1.id, 'utf8')).toBe('1234 MOD'); + })); + + it('should handle resource conflicts', (async () => { + { + const tempFile = tempFilePath('txt'); + await shim.fsDriver().writeFile(tempFile, '1234', 'utf8'); + const folder1 = await Folder.save({ title: 'folder1' }); + const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id }); + await shim.attachFileToNote(note1, tempFile); + await synchronizerStart(); + } + + await switchClient(2); + + { + await synchronizerStart(); + await resourceFetcher().start(); + await resourceFetcher().waitForAllFinished(); + const resource = (await Resource.all())[0]; + const modFile2 = tempFilePath('txt'); + await shim.fsDriver().writeFile(modFile2, '1234 MOD 2', 'utf8'); + await Resource.updateResourceBlobContent(resource.id, modFile2); + await synchronizerStart(); + } + + await switchClient(1); + + { + // Going to modify a resource without syncing first, which will cause a conflict + const resource = (await Resource.all())[0]; + const modFile1 = tempFilePath('txt'); + await shim.fsDriver().writeFile(modFile1, '1234 MOD 1', 'utf8'); + await Resource.updateResourceBlobContent(resource.id, modFile1); + await synchronizerStart(); // CONFLICT + + // If we try to read the resource content now, it should throw because the local + // content has been moved to the conflict notebook, and the new local content + // has not been downloaded yet. + await checkThrowAsync(async () => await Resource.resourceBlobContent(resource.id)); + + // Now download resources, and our local content would have been overwritten by + // the content from client 2 + await resourceFetcher().start(); + await resourceFetcher().waitForAllFinished(); + const localContent = await Resource.resourceBlobContent(resource.id, 'utf8'); + expect(localContent).toBe('1234 MOD 2'); + + // Check that the Conflict note has been generated, with the conflict resource + // attached to it, and check that it has the original content. + const allNotes = await Note.all(); + expect(allNotes.length).toBe(2); + const conflictNote = allNotes.find((v: NoteEntity) => { + return !!v.is_conflict; + }); + expect(!!conflictNote).toBe(true); + const resourceIds = await Note.linkedResourceIds(conflictNote.body); + expect(resourceIds.length).toBe(1); + const conflictContent = await Resource.resourceBlobContent(resourceIds[0], 'utf8'); + expect(conflictContent).toBe('1234 MOD 1'); + } + })); + + it('should handle resource conflicts if a resource is changed locally but deleted remotely', (async () => { + { + const tempFile = tempFilePath('txt'); + await shim.fsDriver().writeFile(tempFile, '1234', 'utf8'); + const folder1 = await Folder.save({ title: 'folder1' }); + const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id }); + await shim.attachFileToNote(note1, tempFile); + await synchronizerStart(); + } + + await switchClient(2); + + { + await synchronizerStart(); + await resourceFetcher().start(); + await resourceFetcher().waitForAllFinished(); + } + + await switchClient(1); + + { + const resource = (await Resource.all())[0]; + await Resource.delete(resource.id); + await synchronizerStart(); + + } + + await switchClient(2); + + { + const originalResource = (await Resource.all())[0]; + await Resource.save({ id: originalResource.id, title: 'modified resource' }); + await synchronizerStart(); // CONFLICT + + const deletedResource = await Resource.load(originalResource.id); + expect(!deletedResource).toBe(true); + + const allResources = await Resource.all(); + expect(allResources.length).toBe(1); + const conflictResource = allResources[0]; + expect(originalResource.id).not.toBe(conflictResource.id); + expect(conflictResource.title).toBe('modified resource'); + } + })); + + it('should not upload a resource if it has not been fetched yet', (async () => { + // In some rare cases, the synchronizer might try to upload a resource even though it + // doesn't have the resource file. It can happen in this situation: + // - C1 create resource + // - C1 sync + // - C2 sync + // - C2 resource metadata is received but ResourceFetcher hasn't downloaded the file yet + // - C2 enables E2EE - all the items are marked for forced sync + // - C2 sync + // The synchronizer will try to upload the resource, even though it doesn't have the file, + // so we need to make sure it doesn't. But also that once it gets the file, the resource + // does get uploaded. + + const note1 = await Note.save({ title: 'note' }); + await shim.attachFileToNote(note1, `${__dirname}/../tests/support/photo.jpg`); + const resource = (await Resource.all())[0]; + await Resource.setLocalState(resource.id, { fetch_status: Resource.FETCH_STATUS_IDLE }); + await synchronizerStart(); + + expect((await remoteResources()).length).toBe(0); + + await Resource.setLocalState(resource.id, { fetch_status: Resource.FETCH_STATUS_DONE }); + await synchronizerStart(); + + expect((await remoteResources()).length).toBe(1); + })); + + it('should not download resources over the limit', (async () => { + const note1 = await Note.save({ title: 'note' }); + await shim.attachFileToNote(note1, `${__dirname}/../tests/support/photo.jpg`); + await synchronizer().start(); + + await switchClient(2); + + const previousMax = synchronizer().maxResourceSize_; + synchronizer().maxResourceSize_ = 1; + await synchronizerStart(); + synchronizer().maxResourceSize_ = previousMax; + + const syncItems = await BaseItem.allSyncItems(syncTargetId()); + expect(syncItems.length).toBe(2); + expect(syncItems[1].item_location).toBe(BaseItem.SYNC_ITEM_LOCATION_REMOTE); + expect(syncItems[1].sync_disabled).toBe(1); + })); + +}); \ No newline at end of file diff --git a/packages/app-cli/tests/Synchronizer.ts b/packages/app-cli/tests/Synchronizer.ts index 5013cfac72..1b7eedbdd4 100644 --- a/packages/app-cli/tests/Synchronizer.ts +++ b/packages/app-cli/tests/Synchronizer.ts @@ -31,40 +31,6 @@ describe('synchronizer', function() { insideBeforeEach = false; }); - it('should allow duplicate folder titles', (async () => { - await Folder.save({ title: 'folder' }); - - await switchClient(2); - - let remoteF2 = await Folder.save({ title: 'folder' }); - await synchronizerStart(); - - await switchClient(1); - - await sleep(0.1); - - await synchronizerStart(); - - const localF2 = await Folder.load(remoteF2.id); - - expect(localF2.title == remoteF2.title).toBe(true); - - // Then that folder that has been renamed locally should be set in such a way - // that synchronizing it applies the title change remotely, and that new title - // should be retrieved by client 2. - - await synchronizerStart(); - - await switchClient(2); - await sleep(0.1); - - await synchronizerStart(); - - remoteF2 = await Folder.load(remoteF2.id); - - expect(remoteF2.title == localF2.title).toBe(true); - })); - async function shoudSyncTagTest(withEncryption: boolean) { let masterKey = null; if (withEncryption) { @@ -122,603 +88,6 @@ describe('synchronizer', function() { await shoudSyncTagTest(true); })); - it('notes and folders should get encrypted when encryption is enabled', (async () => { - Setting.setValue('encryption.enabled', true); - const masterKey = await loadEncryptionMasterKey(); - const folder1 = await Folder.save({ title: 'folder1' }); - let note1 = await Note.save({ title: 'un', body: 'to be encrypted', parent_id: folder1.id }); - await synchronizerStart(); - // After synchronisation, remote items should be encrypted but local ones remain plain text - note1 = await Note.load(note1.id); - expect(note1.title).toBe('un'); - - await switchClient(2); - - await synchronizerStart(); - let folder1_2 = await Folder.load(folder1.id); - let note1_2 = await Note.load(note1.id); - const masterKey_2 = await MasterKey.load(masterKey.id); - // On this side however it should be received encrypted - expect(!note1_2.title).toBe(true); - expect(!folder1_2.title).toBe(true); - expect(!!note1_2.encryption_cipher_text).toBe(true); - expect(!!folder1_2.encryption_cipher_text).toBe(true); - // Master key is already encrypted so it does not get re-encrypted during sync - expect(masterKey_2.content).toBe(masterKey.content); - expect(masterKey_2.checksum).toBe(masterKey.checksum); - // Now load the master key we got from client 1 and try to decrypt - await encryptionService().loadMasterKey_(masterKey_2, '123456', true); - // Get the decrypted items back - await Folder.decrypt(folder1_2); - await Note.decrypt(note1_2); - folder1_2 = await Folder.load(folder1.id); - note1_2 = await Note.load(note1.id); - // Check that properties match the original items. Also check - // the encryption did not affect the updated_time timestamp. - expect(note1_2.title).toBe(note1.title); - expect(note1_2.body).toBe(note1.body); - expect(note1_2.updated_time).toBe(note1.updated_time); - expect(!note1_2.encryption_cipher_text).toBe(true); - expect(folder1_2.title).toBe(folder1.title); - expect(folder1_2.updated_time).toBe(folder1.updated_time); - expect(!folder1_2.encryption_cipher_text).toBe(true); - })); - - it('should enable encryption automatically when downloading new master key (and none was previously available)',(async () => { - // Enable encryption on client 1 and sync an item - Setting.setValue('encryption.enabled', true); - await loadEncryptionMasterKey(); - let folder1 = await Folder.save({ title: 'folder1' }); - await synchronizerStart(); - - await switchClient(2); - - // Synchronising should enable encryption since we're going to get a master key - expect(Setting.value('encryption.enabled')).toBe(false); - await synchronizerStart(); - expect(Setting.value('encryption.enabled')).toBe(true); - - // Check that we got the master key from client 1 - const masterKey = (await MasterKey.all())[0]; - expect(!!masterKey).toBe(true); - - // Since client 2 hasn't supplied a password yet, no master key is currently loaded - expect(encryptionService().loadedMasterKeyIds().length).toBe(0); - - // If we sync now, nothing should be sent to target since we don't have a password. - // Technically it's incorrect to set the property of an encrypted variable but it allows confirming - // that encryption doesn't work if user hasn't supplied a password. - await BaseItem.forceSync(folder1.id); - await synchronizerStart(); - - await switchClient(1); - - await synchronizerStart(); - folder1 = await Folder.load(folder1.id); - expect(folder1.title).toBe('folder1'); // Still at old value - - await switchClient(2); - - // Now client 2 set the master key password - Setting.setObjectValue('encryption.passwordCache', masterKey.id, '123456'); - await encryptionService().loadMasterKeysFromSettings(); - - // Now that master key should be loaded - expect(encryptionService().loadedMasterKeyIds()[0]).toBe(masterKey.id); - - // Decrypt all the data. Now change the title and sync again - this time the changes should be transmitted - await decryptionWorker().start(); - await Folder.save({ id: folder1.id, title: 'change test' }); - - // If we sync now, this time client 1 should get the changes we did earlier - await synchronizerStart(); - - await switchClient(1); - - await synchronizerStart(); - // Decrypt the data we just got - await decryptionWorker().start(); - folder1 = await Folder.load(folder1.id); - expect(folder1.title).toBe('change test'); // Got title from client 2 - })); - - it('should encrypt existing notes too when enabling E2EE', (async () => { - // First create a folder, without encryption enabled, and sync it - await Folder.save({ title: 'folder1' }); - await synchronizerStart(); - let files = await fileApi().list('', { includeDirs: false, syncItemsOnly: true }); - let content = await fileApi().get(files.items[0].path); - expect(content.indexOf('folder1') >= 0).toBe(true); - - // Then enable encryption and sync again - let masterKey = await encryptionService().generateMasterKey('123456'); - masterKey = await MasterKey.save(masterKey); - await encryptionService().enableEncryption(masterKey, '123456'); - await encryptionService().loadMasterKeysFromSettings(); - await synchronizerStart(); - - // Even though the folder has not been changed it should have been synced again so that - // an encrypted version of it replaces the decrypted version. - files = await fileApi().list('', { includeDirs: false, syncItemsOnly: true }); - expect(files.items.length).toBe(2); - // By checking that the folder title is not present, we can confirm that the item has indeed been encrypted - // One of the two items is the master key - content = await fileApi().get(files.items[0].path); - expect(content.indexOf('folder1') < 0).toBe(true); - content = await fileApi().get(files.items[1].path); - expect(content.indexOf('folder1') < 0).toBe(true); - })); - - it('should sync resources', (async () => { - while (insideBeforeEach) await time.msleep(500); - - const folder1 = await Folder.save({ title: 'folder1' }); - const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id }); - await shim.attachFileToNote(note1, `${__dirname}/../tests/support/photo.jpg`); - const resource1 = (await Resource.all())[0]; - const resourcePath1 = Resource.fullPath(resource1); - await synchronizerStart(); - expect((await remoteNotesFoldersResources()).length).toBe(3); - - await switchClient(2); - - await synchronizerStart(); - const allResources = await Resource.all(); - expect(allResources.length).toBe(1); - let resource1_2 = allResources[0]; - let ls = await Resource.localState(resource1_2); - expect(resource1_2.id).toBe(resource1.id); - expect(ls.fetch_status).toBe(Resource.FETCH_STATUS_IDLE); - - const fetcher = new ResourceFetcher(() => { return synchronizer().api(); }); - fetcher.queueDownload_(resource1_2.id); - await fetcher.waitForAllFinished(); - - resource1_2 = await Resource.load(resource1.id); - ls = await Resource.localState(resource1_2); - expect(ls.fetch_status).toBe(Resource.FETCH_STATUS_DONE); - - const resourcePath1_2 = Resource.fullPath(resource1_2); - expect(fileContentEqual(resourcePath1, resourcePath1_2)).toBe(true); - })); - - it('should handle resource download errors', (async () => { - while (insideBeforeEach) await time.msleep(500); - - const folder1 = await Folder.save({ title: 'folder1' }); - const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id }); - await shim.attachFileToNote(note1, `${__dirname}/../tests/support/photo.jpg`); - let resource1 = (await Resource.all())[0]; - await synchronizerStart(); - - await switchClient(2); - - await synchronizerStart(); - - const fetcher = new ResourceFetcher(() => { - return { - // Simulate a failed download - get: () => { return new Promise((_resolve: Function, reject: Function) => { reject(new Error('did not work')); }); }, - }; - }); - fetcher.queueDownload_(resource1.id); - await fetcher.waitForAllFinished(); - - resource1 = await Resource.load(resource1.id); - const ls = await Resource.localState(resource1); - expect(ls.fetch_status).toBe(Resource.FETCH_STATUS_ERROR); - expect(ls.fetch_error).toBe('did not work'); - })); - - it('should set the resource file size if it is missing', (async () => { - while (insideBeforeEach) await time.msleep(500); - - const folder1 = await Folder.save({ title: 'folder1' }); - const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id }); - await shim.attachFileToNote(note1, `${__dirname}/../tests/support/photo.jpg`); - await synchronizerStart(); - - await switchClient(2); - - await synchronizerStart(); - let r1 = (await Resource.all())[0]; - await Resource.setFileSizeOnly(r1.id, -1); - r1 = await Resource.load(r1.id); - expect(r1.size).toBe(-1); - - const fetcher = new ResourceFetcher(() => { return synchronizer().api(); }); - fetcher.queueDownload_(r1.id); - await fetcher.waitForAllFinished(); - r1 = await Resource.load(r1.id); - expect(r1.size).toBe(2720); - })); - - it('should delete resources', (async () => { - while (insideBeforeEach) await time.msleep(500); - - const folder1 = await Folder.save({ title: 'folder1' }); - const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id }); - await shim.attachFileToNote(note1, `${__dirname}/../tests/support/photo.jpg`); - const resource1 = (await Resource.all())[0]; - const resourcePath1 = Resource.fullPath(resource1); - await synchronizerStart(); - - await switchClient(2); - - await synchronizerStart(); - let allResources = await Resource.all(); - expect(allResources.length).toBe(1); - expect((await remoteNotesFoldersResources()).length).toBe(3); - await Resource.delete(resource1.id); - await synchronizerStart(); - expect((await remoteNotesFoldersResources()).length).toBe(2); - - const remoteBlob = await fileApi().stat(`.resource/${resource1.id}`); - expect(!remoteBlob).toBe(true); - - await switchClient(1); - - expect(await shim.fsDriver().exists(resourcePath1)).toBe(true); - await synchronizerStart(); - allResources = await Resource.all(); - expect(allResources.length).toBe(0); - expect(await shim.fsDriver().exists(resourcePath1)).toBe(false); - })); - - it('should encrypt resources', (async () => { - Setting.setValue('encryption.enabled', true); - const masterKey = await loadEncryptionMasterKey(); - - const folder1 = await Folder.save({ title: 'folder1' }); - const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id }); - await shim.attachFileToNote(note1, `${__dirname}/../tests/support/photo.jpg`); - const resource1 = (await Resource.all())[0]; - const resourcePath1 = Resource.fullPath(resource1); - await synchronizerStart(); - - await switchClient(2); - - await synchronizerStart(); - Setting.setObjectValue('encryption.passwordCache', masterKey.id, '123456'); - await encryptionService().loadMasterKeysFromSettings(); - - const fetcher = new ResourceFetcher(() => { return synchronizer().api(); }); - fetcher.queueDownload_(resource1.id); - await fetcher.waitForAllFinished(); - - let resource1_2 = (await Resource.all())[0]; - resource1_2 = await Resource.decrypt(resource1_2); - const resourcePath1_2 = Resource.fullPath(resource1_2); - - expect(fileContentEqual(resourcePath1, resourcePath1_2)).toBe(true); - })); - - it('should sync resource blob changes', (async () => { - const tempFile = tempFilePath('txt'); - await shim.fsDriver().writeFile(tempFile, '1234', 'utf8'); - const folder1 = await Folder.save({ title: 'folder1' }); - const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id }); - await shim.attachFileToNote(note1, tempFile); - await synchronizerStart(); - - await switchClient(2); - - await synchronizerStart(); - await resourceFetcher().start(); - await resourceFetcher().waitForAllFinished(); - let resource1_2 = (await Resource.all())[0]; - const modFile = tempFilePath('txt'); - await shim.fsDriver().writeFile(modFile, '1234 MOD', 'utf8'); - await Resource.updateResourceBlobContent(resource1_2.id, modFile); - const originalSize = resource1_2.size; - resource1_2 = (await Resource.all())[0]; - const newSize = resource1_2.size; - expect(originalSize).toBe(4); - expect(newSize).toBe(8); - await synchronizerStart(); - - await switchClient(1); - - await synchronizerStart(); - await resourceFetcher().start(); - await resourceFetcher().waitForAllFinished(); - const resource1_1 = (await Resource.all())[0]; - expect(resource1_1.size).toBe(newSize); - expect(await Resource.resourceBlobContent(resource1_1.id, 'utf8')).toBe('1234 MOD'); - })); - - it('should handle resource conflicts', (async () => { - { - const tempFile = tempFilePath('txt'); - await shim.fsDriver().writeFile(tempFile, '1234', 'utf8'); - const folder1 = await Folder.save({ title: 'folder1' }); - const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id }); - await shim.attachFileToNote(note1, tempFile); - await synchronizerStart(); - } - - await switchClient(2); - - { - await synchronizerStart(); - await resourceFetcher().start(); - await resourceFetcher().waitForAllFinished(); - const resource = (await Resource.all())[0]; - const modFile2 = tempFilePath('txt'); - await shim.fsDriver().writeFile(modFile2, '1234 MOD 2', 'utf8'); - await Resource.updateResourceBlobContent(resource.id, modFile2); - await synchronizerStart(); - } - - await switchClient(1); - - { - // Going to modify a resource without syncing first, which will cause a conflict - const resource = (await Resource.all())[0]; - const modFile1 = tempFilePath('txt'); - await shim.fsDriver().writeFile(modFile1, '1234 MOD 1', 'utf8'); - await Resource.updateResourceBlobContent(resource.id, modFile1); - await synchronizerStart(); // CONFLICT - - // If we try to read the resource content now, it should throw because the local - // content has been moved to the conflict notebook, and the new local content - // has not been downloaded yet. - await checkThrowAsync(async () => await Resource.resourceBlobContent(resource.id)); - - // Now download resources, and our local content would have been overwritten by - // the content from client 2 - await resourceFetcher().start(); - await resourceFetcher().waitForAllFinished(); - const localContent = await Resource.resourceBlobContent(resource.id, 'utf8'); - expect(localContent).toBe('1234 MOD 2'); - - // Check that the Conflict note has been generated, with the conflict resource - // attached to it, and check that it has the original content. - const allNotes = await Note.all(); - expect(allNotes.length).toBe(2); - const conflictNote = allNotes.find((v: NoteEntity) => { - return !!v.is_conflict; - }); - expect(!!conflictNote).toBe(true); - const resourceIds = await Note.linkedResourceIds(conflictNote.body); - expect(resourceIds.length).toBe(1); - const conflictContent = await Resource.resourceBlobContent(resourceIds[0], 'utf8'); - expect(conflictContent).toBe('1234 MOD 1'); - } - })); - - it('should handle resource conflicts if a resource is changed locally but deleted remotely', (async () => { - { - const tempFile = tempFilePath('txt'); - await shim.fsDriver().writeFile(tempFile, '1234', 'utf8'); - const folder1 = await Folder.save({ title: 'folder1' }); - const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id }); - await shim.attachFileToNote(note1, tempFile); - await synchronizerStart(); - } - - await switchClient(2); - - { - await synchronizerStart(); - await resourceFetcher().start(); - await resourceFetcher().waitForAllFinished(); - } - - await switchClient(1); - - { - const resource = (await Resource.all())[0]; - await Resource.delete(resource.id); - await synchronizerStart(); - - } - - await switchClient(2); - - { - const originalResource = (await Resource.all())[0]; - await Resource.save({ id: originalResource.id, title: 'modified resource' }); - await synchronizerStart(); // CONFLICT - - const deletedResource = await Resource.load(originalResource.id); - expect(!deletedResource).toBe(true); - - const allResources = await Resource.all(); - expect(allResources.length).toBe(1); - const conflictResource = allResources[0]; - expect(originalResource.id).not.toBe(conflictResource.id); - expect(conflictResource.title).toBe('modified resource'); - } - })); - - it('should upload decrypted items to sync target after encryption disabled', (async () => { - Setting.setValue('encryption.enabled', true); - await loadEncryptionMasterKey(); - - await Folder.save({ title: 'folder1' }); - await synchronizerStart(); - - let allEncrypted = await allSyncTargetItemsEncrypted(); - expect(allEncrypted).toBe(true); - - await encryptionService().disableEncryption(); - - await synchronizerStart(); - allEncrypted = await allSyncTargetItemsEncrypted(); - expect(allEncrypted).toBe(false); - })); - - it('should not upload any item if encryption was enabled, and items have not been decrypted, and then encryption disabled', (async () => { - // For some reason I can't explain, this test is sometimes executed before beforeEach is finished - // which means it's going to fail in unexpected way. So the loop below wait for beforeEach to be done. - while (insideBeforeEach) await time.msleep(100); - - Setting.setValue('encryption.enabled', true); - const masterKey = await loadEncryptionMasterKey(); - - await Folder.save({ title: 'folder1' }); - await synchronizerStart(); - - await switchClient(2); - - await synchronizerStart(); - expect(Setting.value('encryption.enabled')).toBe(true); - - // If we try to disable encryption now, it should throw an error because some items are - // currently encrypted. They must be decrypted first so that they can be sent as - // plain text to the sync target. - // let hasThrown = await checkThrowAsync(async () => await encryptionService().disableEncryption()); - // expect(hasThrown).toBe(true); - - // Now supply the password, and decrypt the items - Setting.setObjectValue('encryption.passwordCache', masterKey.id, '123456'); - await encryptionService().loadMasterKeysFromSettings(); - await decryptionWorker().start(); - - // Try to disable encryption again - const hasThrown = await checkThrowAsync(async () => await encryptionService().disableEncryption()); - expect(hasThrown).toBe(false); - - // If we sync now the target should receive the decrypted items - await synchronizerStart(); - const allEncrypted = await allSyncTargetItemsEncrypted(); - expect(allEncrypted).toBe(false); - })); - - it('should set the resource file size after decryption', (async () => { - Setting.setValue('encryption.enabled', true); - const masterKey = await loadEncryptionMasterKey(); - - const folder1 = await Folder.save({ title: 'folder1' }); - const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id }); - await shim.attachFileToNote(note1, `${__dirname}/../tests/support/photo.jpg`); - const resource1 = (await Resource.all())[0]; - await Resource.setFileSizeOnly(resource1.id, -1); - Resource.fullPath(resource1); - await synchronizerStart(); - - await switchClient(2); - - await synchronizerStart(); - Setting.setObjectValue('encryption.passwordCache', masterKey.id, '123456'); - await encryptionService().loadMasterKeysFromSettings(); - - const fetcher = new ResourceFetcher(() => { return synchronizer().api(); }); - fetcher.queueDownload_(resource1.id); - await fetcher.waitForAllFinished(); - await decryptionWorker().start(); - - const resource1_2 = await Resource.load(resource1.id); - expect(resource1_2.size).toBe(2720); - })); - - it('should encrypt remote resources after encryption has been enabled', (async () => { - while (insideBeforeEach) await time.msleep(100); - - const folder1 = await Folder.save({ title: 'folder1' }); - const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id }); - await shim.attachFileToNote(note1, `${__dirname}/../tests/support/photo.jpg`); - await synchronizerStart(); - - expect(await allSyncTargetItemsEncrypted()).toBe(false); - - const masterKey = await loadEncryptionMasterKey(); - await encryptionService().enableEncryption(masterKey, '123456'); - await encryptionService().loadMasterKeysFromSettings(); - - await synchronizerStart(); - - expect(await allSyncTargetItemsEncrypted()).toBe(true); - })); - - it('should upload encrypted resource, but it should not mark the blob as encrypted locally', (async () => { - while (insideBeforeEach) await time.msleep(100); - - const folder1 = await Folder.save({ title: 'folder1' }); - const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id }); - await shim.attachFileToNote(note1, `${__dirname}/../tests/support/photo.jpg`); - const masterKey = await loadEncryptionMasterKey(); - await encryptionService().enableEncryption(masterKey, '123456'); - await encryptionService().loadMasterKeysFromSettings(); - await synchronizerStart(); - - const resource1 = (await Resource.all())[0]; - expect(resource1.encryption_blob_encrypted).toBe(0); - })); - - it('should create remote items with UTF-8 content', (async () => { - const folder = await Folder.save({ title: 'Fahrräder' }); - await Note.save({ title: 'Fahrräder', body: 'Fahrräder', parent_id: folder.id }); - const all = await allNotesFolders(); - - await synchronizerStart(); - - await localNotesFoldersSameAsRemote(all, expect); - })); - - it('should update remote items but not pull remote changes', (async () => { - const folder = await Folder.save({ title: 'folder1' }); - const note = await Note.save({ title: 'un', parent_id: folder.id }); - await synchronizerStart(); - - await switchClient(2); - - await synchronizerStart(); - await Note.save({ title: 'deux', parent_id: folder.id }); - await synchronizerStart(); - - await switchClient(1); - - await Note.save({ title: 'un UPDATE', id: note.id }); - await synchronizerStart(null, { syncSteps: ['update_remote'] }); - const all = await allNotesFolders(); - expect(all.length).toBe(2); - - await switchClient(2); - - await synchronizerStart(); - const note2 = await Note.load(note.id); - expect(note2.title).toBe('un UPDATE'); - })); - - it('should create a new Welcome notebook on each client', (async () => { - // Create the Welcome items on two separate clients - - await WelcomeUtils.createWelcomeItems(); - await synchronizerStart(); - - await switchClient(2); - - await WelcomeUtils.createWelcomeItems(); - const beforeFolderCount = (await Folder.all()).length; - const beforeNoteCount = (await Note.all()).length; - expect(beforeFolderCount === 1).toBe(true); - expect(beforeNoteCount > 1).toBe(true); - - await synchronizerStart(); - - const afterFolderCount = (await Folder.all()).length; - const afterNoteCount = (await Note.all()).length; - - expect(afterFolderCount).toBe(beforeFolderCount * 2); - expect(afterNoteCount).toBe(beforeNoteCount * 2); - - // Changes to the Welcome items should be synced to all clients - - const f1 = (await Folder.all())[0]; - await Folder.save({ id: f1.id, title: 'Welcome MOD' }); - - await synchronizerStart(); - - await switchClient(1); - - await synchronizerStart(); - - const f1_1 = await Folder.load(f1.id); - expect(f1_1.title).toBe('Welcome MOD'); - })); - it('should not save revisions when updating a note via sync', (async () => { // When a note is updated, a revision of the original is created. // Here, on client 1, the note is updated for the first time, however since it is @@ -851,85 +220,6 @@ describe('synchronizer', function() { expect((await revisionService().revisionNote(revisions, 1)).title).toBe('note REV2'); })); - it('should not download resources over the limit', (async () => { - const note1 = await Note.save({ title: 'note' }); - await shim.attachFileToNote(note1, `${__dirname}/../tests/support/photo.jpg`); - await synchronizer().start(); - - await switchClient(2); - - const previousMax = synchronizer().maxResourceSize_; - synchronizer().maxResourceSize_ = 1; - await synchronizerStart(); - synchronizer().maxResourceSize_ = previousMax; - - const syncItems = await BaseItem.allSyncItems(syncTargetId()); - expect(syncItems.length).toBe(2); - expect(syncItems[1].item_location).toBe(BaseItem.SYNC_ITEM_LOCATION_REMOTE); - expect(syncItems[1].sync_disabled).toBe(1); - })); - - it('should not upload a resource if it has not been fetched yet', (async () => { - // In some rare cases, the synchronizer might try to upload a resource even though it - // doesn't have the resource file. It can happen in this situation: - // - C1 create resource - // - C1 sync - // - C2 sync - // - C2 resource metadata is received but ResourceFetcher hasn't downloaded the file yet - // - C2 enables E2EE - all the items are marked for forced sync - // - C2 sync - // The synchronizer will try to upload the resource, even though it doesn't have the file, - // so we need to make sure it doesn't. But also that once it gets the file, the resource - // does get uploaded. - - const note1 = await Note.save({ title: 'note' }); - await shim.attachFileToNote(note1, `${__dirname}/../tests/support/photo.jpg`); - const resource = (await Resource.all())[0]; - await Resource.setLocalState(resource.id, { fetch_status: Resource.FETCH_STATUS_IDLE }); - await synchronizerStart(); - - expect((await remoteResources()).length).toBe(0); - - await Resource.setLocalState(resource.id, { fetch_status: Resource.FETCH_STATUS_DONE }); - await synchronizerStart(); - - expect((await remoteResources()).length).toBe(1); - })); - - it('should decrypt the resource metadata, but not try to decrypt the file, if it is not present', (async () => { - const note1 = await Note.save({ title: 'note' }); - await shim.attachFileToNote(note1, `${__dirname}/../tests/support/photo.jpg`); - const masterKey = await loadEncryptionMasterKey(); - await encryptionService().enableEncryption(masterKey, '123456'); - await encryptionService().loadMasterKeysFromSettings(); - await synchronizerStart(); - expect(await allSyncTargetItemsEncrypted()).toBe(true); - - await switchClient(2); - - await synchronizerStart(); - Setting.setObjectValue('encryption.passwordCache', masterKey.id, '123456'); - await encryptionService().loadMasterKeysFromSettings(); - await decryptionWorker().start(); - - let resource = (await Resource.all())[0]; - - expect(!!resource.encryption_applied).toBe(false); - expect(!!resource.encryption_blob_encrypted).toBe(true); - - const resourceFetcher = new ResourceFetcher(() => { return synchronizer().api(); }); - await resourceFetcher.start(); - await resourceFetcher.waitForAllFinished(); - - const ls = await Resource.localState(resource); - expect(ls.fetch_status).toBe(Resource.FETCH_STATUS_DONE); - - await decryptionWorker().start(); - resource = (await Resource.all())[0]; - - expect(!!resource.encryption_blob_encrypted).toBe(false); - })); - it('should not create revisions when item is modified as a result of decryption', (async () => { // Handle this scenario: // - C1 creates note @@ -966,130 +256,4 @@ describe('synchronizer', function() { expect((await Revision.all()).length).toBe(0); })); - it('should stop trying to decrypt item after a few attempts', (async () => { - let hasThrown; - - const note = await Note.save({ title: 'ma note' }); - const masterKey = await loadEncryptionMasterKey(); - await encryptionService().enableEncryption(masterKey, '123456'); - await encryptionService().loadMasterKeysFromSettings(); - await synchronizerStart(); - - await switchClient(2); - - await synchronizerStart(); - - // First, simulate a broken note and check that the decryption worker - // gives up decrypting after a number of tries. This is mainly relevant - // for data that crashes the mobile application - we don't want to keep - // decrypting these. - - const encryptedNote = await Note.load(note.id); - const goodCipherText = encryptedNote.encryption_cipher_text; - await Note.save({ id: note.id, encryption_cipher_text: 'doesntlookright' }); - - Setting.setObjectValue('encryption.passwordCache', masterKey.id, '123456'); - await encryptionService().loadMasterKeysFromSettings(); - - hasThrown = await checkThrowAsync(async () => await decryptionWorker().start({ errorHandler: 'throw' })); - expect(hasThrown).toBe(true); - - hasThrown = await checkThrowAsync(async () => await decryptionWorker().start({ errorHandler: 'throw' })); - expect(hasThrown).toBe(true); - - // Third time, an error is logged and no error is thrown - hasThrown = await checkThrowAsync(async () => await decryptionWorker().start({ errorHandler: 'throw' })); - expect(hasThrown).toBe(false); - - const disabledItems = await decryptionWorker().decryptionDisabledItems(); - expect(disabledItems.length).toBe(1); - expect(disabledItems[0].id).toBe(note.id); - - expect((await kvStore().all()).length).toBe(1); - await kvStore().clear(); - - // Now check that if it fails once but succeed the second time, the note - // is correctly decrypted and the counters are cleared. - - hasThrown = await checkThrowAsync(async () => await decryptionWorker().start({ errorHandler: 'throw' })); - expect(hasThrown).toBe(true); - - await Note.save({ id: note.id, encryption_cipher_text: goodCipherText }); - - hasThrown = await checkThrowAsync(async () => await decryptionWorker().start({ errorHandler: 'throw' })); - expect(hasThrown).toBe(false); - - const decryptedNote = await Note.load(note.id); - expect(decryptedNote.title).toBe('ma note'); - - expect((await kvStore().all()).length).toBe(0); - expect((await decryptionWorker().decryptionDisabledItems()).length).toBe(0); - })); - - it('should not wipe out user data when syncing with an empty target', (async () => { - // Only these targets support the wipeOutFailSafe flag (in other words, the targets that use basicDelta) - if (!['nextcloud', 'memory', 'filesystem', 'amazon_s3'].includes(syncTargetName())) return; - - for (let i = 0; i < 10; i++) await Note.save({ title: 'note' }); - - Setting.setValue('sync.wipeOutFailSafe', true); - await synchronizerStart(); - await fileApi().clearRoot(); // oops - await synchronizerStart(); - expect((await Note.all()).length).toBe(10); // but since the fail-safe if on, the notes have not been deleted - - Setting.setValue('sync.wipeOutFailSafe', false); // Now switch it off - await synchronizerStart(); - expect((await Note.all()).length).toBe(0); // Since the fail-safe was off, the data has been cleared - - // Handle case where the sync target has been wiped out, then the user creates one note and sync. - - for (let i = 0; i < 10; i++) await Note.save({ title: 'note' }); - Setting.setValue('sync.wipeOutFailSafe', true); - await synchronizerStart(); - await fileApi().clearRoot(); - await Note.save({ title: 'ma note encore' }); - await synchronizerStart(); - expect((await Note.all()).length).toBe(11); - })); - - it('should not encrypt notes that are shared', (async () => { - Setting.setValue('encryption.enabled', true); - await loadEncryptionMasterKey(); - - const folder1 = await Folder.save({ title: 'folder1' }); - const note1 = await Note.save({ title: 'un', parent_id: folder1.id }); - let note2 = await Note.save({ title: 'deux', parent_id: folder1.id }); - await synchronizerStart(); - - await switchClient(2); - - await synchronizerStart(); - - await switchClient(1); - - const origNote2 = Object.assign({}, note2); - await BaseItem.updateShareStatus(note2, true); - note2 = await Note.load(note2.id); - - // Sharing a note should not modify the timestamps - expect(note2.user_updated_time).toBe(origNote2.user_updated_time); - expect(note2.user_created_time).toBe(origNote2.user_created_time); - - await synchronizerStart(); - - await switchClient(2); - - await synchronizerStart(); - - // The shared note should be decrypted - const note2_2 = await Note.load(note2.id); - expect(note2_2.title).toBe('deux'); - expect(note2_2.is_shared).toBe(1); - - // The non-shared note should be encrypted - const note1_2 = await Note.load(note1.id); - expect(note1_2.title).toBe(''); - })); - });