From c19f9eb7059595da7d5f3f0e50e8518578927b18 Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Wed, 18 Oct 2023 17:54:29 +0100 Subject: [PATCH] update --- packages/app-mobile/ios/Podfile.lock | 12 +- packages/server/package.json | 1 + .../src/migrations/20190913171451_create.ts | 2 +- packages/server/src/models/UserModel.ts | 19 +- packages/server/src/utils/array.ts | 2 +- packages/server/src/utils/auth.test.ts | 2 +- packages/server/src/utils/auth.ts | 12 +- .../src/utils/testing/populateDatabase.ts | 294 ++++++++++++------ .../server/src/utils/testing/testUtils.ts | 11 +- yarn.lock | 8 + 10 files changed, 251 insertions(+), 112 deletions(-) diff --git a/packages/app-mobile/ios/Podfile.lock b/packages/app-mobile/ios/Podfile.lock index d8ba953fbc..f2e85f18f1 100644 --- a/packages/app-mobile/ios/Podfile.lock +++ b/packages/app-mobile/ios/Podfile.lock @@ -288,7 +288,7 @@ PODS: - React-Core - react-native-get-random-values (1.9.0): - React-Core - - react-native-image-picker (5.6.1): + - react-native-image-picker (5.7.0): - React-Core - react-native-image-resizer (3.0.7): - React-Core @@ -418,11 +418,11 @@ PODS: - React-Core - RNVectorIcons (10.0.0): - React-Core - - RNZipArchive (6.0.9): + - RNZipArchive (6.1.0): - React-Core - - RNZipArchive/Core (= 6.0.9) + - RNZipArchive/Core (= 6.1.0) - SSZipArchive (~> 2.2) - - RNZipArchive/Core (6.0.9): + - RNZipArchive/Core (6.1.0): - React-Core - SSZipArchive (~> 2.2) - SSZipArchive (2.4.3) @@ -669,7 +669,7 @@ SPEC CHECKSUMS: react-native-fingerprint-scanner: ac6656f18c8e45a7459302b84da41a44ad96dbbe react-native-geolocation: 0f7fe8a4c2de477e278b0365cce27d089a8c5903 react-native-get-random-values: dee677497c6a740b71e5612e8dbd83e7539ed5bb - react-native-image-picker: 5fcac5a5ffcb3737837f0617d43fd767249290de + react-native-image-picker: 3269f75c251cdcd61ab51b911dd30d6fff8c6169 react-native-image-resizer: 681f7607418b97c084ba2d0999b153b103040d8a react-native-netinfo: fefd4e98d75cbdd6e85fc530f7111a8afdf2b0c5 react-native-rsa-native: 12132eb627797529fdb1f0d22fd0f8f9678df64a @@ -705,7 +705,7 @@ SPEC CHECKSUMS: RNSecureRandom: 07efbdf2cd99efe13497433668e54acd7df49fef RNShare: 32e97adc8d8c97d4a26bcdd3c45516882184f8b6 RNVectorIcons: 8b5bb0fa61d54cd2020af4f24a51841ce365c7e9 - RNZipArchive: 68a0c6db4b1c103f846f1559622050df254a3ade + RNZipArchive: ef9451b849c45a29509bf44e65b788829ab07801 SSZipArchive: fe6a26b2a54d5a0890f2567b5cc6de5caa600aef Yoga: e7ea9e590e27460d28911403b894722354d73479 diff --git a/packages/server/package.json b/packages/server/package.json index c05d6cac0f..43b23c25e6 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -61,6 +61,7 @@ "devDependencies": { "@joplin/tools": "~2.13", "@rmp135/sql-ts": "1.18.0", + "@types/bcryptjs": "2.4.5", "@types/formidable": "3.4.3", "@types/fs-extra": "11.0.2", "@types/jest": "29.5.4", diff --git a/packages/server/src/migrations/20190913171451_create.ts b/packages/server/src/migrations/20190913171451_create.ts index 8d0edf2311..8121e39aa3 100644 --- a/packages/server/src/migrations/20190913171451_create.ts +++ b/packages/server/src/migrations/20190913171451_create.ts @@ -98,7 +98,7 @@ export const up = async (db: DbConnection) => { await db('users').insert({ id: adminId, email: defaultAdminEmail, - password: hashPassword(defaultAdminPassword), + password: await hashPassword(defaultAdminPassword), full_name: 'Admin', is_admin: 1, updated_time: now, diff --git a/packages/server/src/models/UserModel.ts b/packages/server/src/models/UserModel.ts index 175d662785..a20504c117 100644 --- a/packages/server/src/models/UserModel.ts +++ b/packages/server/src/models/UserModel.ts @@ -125,7 +125,7 @@ export default class UserModel extends BaseModel { public async login(email: string, password: string): Promise { const user = await this.loadByEmail(email); if (!user) return null; - if (!checkPassword(password, user.password)) return null; + if (!(await checkPassword(password, user.password))) return null; return user; } @@ -635,16 +635,23 @@ export default class UserModel extends BaseModel { public async save(object: User, options: SaveOptions = {}): Promise { const user = this.formatValues(object); + const isNew = await this.isNew(object, options); + if (user.password) { if (isHashedPassword(user.password)) { - throw new ErrorBadRequest(`Unable to save user because password already seems to be hashed. User id: ${user.id}`); + if (!isNew) { + throw new ErrorBadRequest(`Unable to save user because password already seems to be hashed. User id: ${user.id}`); + } else { + // OK - We allow supplying an already hashed password for + // new users. This is mostly used for testing, because + // generating a bcrypt hash for each user is slow. + } + } else { + if (!options.skipValidation) this.validatePassword(user.password); + user.password = await hashPassword(user.password); } - if (!options.skipValidation) this.validatePassword(user.password); - user.password = hashPassword(user.password); } - const isNew = await this.isNew(object, options); - return this.withTransaction(async () => { const savedUser = await super.save(user, options); diff --git a/packages/server/src/utils/array.ts b/packages/server/src/utils/array.ts index 1fe3f22416..3643a6e41e 100644 --- a/packages/server/src/utils/array.ts +++ b/packages/server/src/utils/array.ts @@ -5,7 +5,7 @@ export function unique(array: any[]): any[] { } export const randomElement = (array: T[]): T => { - if (!array.length) return null; + if (!array || !array.length) return null; return array[Math.floor(Math.random() * array.length)]; }; diff --git a/packages/server/src/utils/auth.test.ts b/packages/server/src/utils/auth.test.ts index 8b69efa3f4..d795147e01 100644 --- a/packages/server/src/utils/auth.test.ts +++ b/packages/server/src/utils/auth.test.ts @@ -12,7 +12,7 @@ describe('hashPassword', () => { '$2a$10$LMKVPiNOWDZhtw9NizNIEuNGLsjOxQAcrwQJ0lnKuiaOtyFgZEnwO', ], )('should return a string that starts with $2a$10 for the password: %', async (plainText) => { - expect(hashPassword(plainText).startsWith('$2a$10')).toBe(true); + expect((await hashPassword(plainText)).startsWith('$2a$10')).toBe(true); }); }); diff --git a/packages/server/src/utils/auth.ts b/packages/server/src/utils/auth.ts index e7bb03d434..e3d86d919c 100644 --- a/packages/server/src/utils/auth.ts +++ b/packages/server/src/utils/auth.ts @@ -1,12 +1,12 @@ -const bcrypt = require('bcryptjs'); +import * as bcrypt from 'bcryptjs'; -export function hashPassword(password: string): string { - const salt = bcrypt.genSaltSync(10); - return bcrypt.hashSync(password, salt); +export async function hashPassword(password: string): Promise { + const salt = await bcrypt.genSalt(10); + return bcrypt.hash(password, salt); } -export function checkPassword(password: string, hash: string): boolean { - return bcrypt.compareSync(password, hash); +export async function checkPassword(password: string, hash: string): Promise { + return bcrypt.compare(password, hash); } export const isHashedPassword = (password: string) => { diff --git a/packages/server/src/utils/testing/populateDatabase.ts b/packages/server/src/utils/testing/populateDatabase.ts index 5549a01be0..22c296075a 100644 --- a/packages/server/src/utils/testing/populateDatabase.ts +++ b/packages/server/src/utils/testing/populateDatabase.ts @@ -1,8 +1,10 @@ -import { NoteEntity } from '@joplin/lib/services/database/types'; +import { FolderEntity, NoteEntity } from '@joplin/lib/services/database/types'; import Logger, { LogLevel, TargetType } from '@joplin/utils/Logger'; -import { randomElement, removeElement } from '../array'; +import { User } from '../../services/database/types'; +import { randomElement } from '../array'; +import { CustomErrorCode } from '../errors'; import { randomWords } from './randomWords'; -import { afterAllTests, beforeAllDb, createdDbPath, createFolder, createNote, createResource, createUserAndSession, deleteItem, randomHash, updateFolder, updateNote, UserAndSession } from './testUtils'; +import { afterAllTests, beforeAllDb, createdDbPath, makeFolderSerializedBody, makeNoteSerializedBody, makeResourceSerializedBody, models, randomHash } from './testUtils'; const { shimInit } = require('@joplin/lib/shim-init-node.js'); const nodeSqlite = require('sqlite3'); @@ -57,114 +59,191 @@ const isDeleteAction = (action: Action) => { return deleteActions.includes(action); }; -type Reaction = (context: Context, sessionId: string)=> Promise; +type Reaction = (context: Context, user: User)=> Promise; const randomInt = (min: number, max: number) => { return Math.floor(Math.random() * (max - min + 1)) + min; }; -const createRandomNote = (sessionId: string, note: NoteEntity = null) => { - return createNote(sessionId, { - id: randomHash(), +const createRandomNote = async (user: User, note: NoteEntity = null) => { + const id = randomHash(); + const itemName = `${id}.md`; + + const serializedBody = makeNoteSerializedBody({ + id, title: randomWords(randomInt(1, 10)), ...note, }); + + const result = await models().item().saveFromRawContent(user, { + name: itemName, + body: Buffer.from(serializedBody), + }); + + if (result[itemName].error) throw result[itemName].error; + + return result[itemName].item; +}; + +const createRandomFolder = async (user: User, folder: FolderEntity = null) => { + const id = randomHash(); + const itemName = `${id}.md`; + + const serializedBody = makeFolderSerializedBody({ + id, + title: randomWords(randomInt(1, 5)), + ...folder, + }); + + const result = await models().item().saveFromRawContent(user, { + name: itemName, + body: Buffer.from(serializedBody), + }); + + if (result[itemName].error) throw result[itemName].error; + + return result[itemName].item; }; const reactions: Record = { - [Action.CreateNote]: async (context, sessionId) => { - const item = await createRandomNote(sessionId); - if (!context.createdNoteIds[sessionId]) context.createdNoteIds[sessionId] = []; - context.createdNoteIds[sessionId].push(item.jop_id); + [Action.CreateNote]: async (context, user) => { + const item = await createRandomNote(user); + if (!context.createdNoteIds[user.id]) context.createdNoteIds[user.id] = []; + context.createdNoteIds[user.id].push(item.jop_id); + return true; }, - [Action.CreateFolder]: async (context, sessionId) => { - const item = await createFolder(sessionId, { - id: randomHash(), - title: randomWords(randomInt(1, 5)), - }); - if (!context.createdFolderIds[sessionId]) context.createdFolderIds[sessionId] = []; - context.createdFolderIds[sessionId].push(item.jop_id); + [Action.CreateFolder]: async (context, user) => { + const item = await createRandomFolder(user); + if (!context.createdFolderIds[user.id]) context.createdFolderIds[user.id] = []; + context.createdFolderIds[user.id].push(item.jop_id); + return true; }, - [Action.CreateNoteAndResource]: async (context, sessionId) => { - const item = await createResource(sessionId, { - id: randomHash(), - title: randomWords(randomInt(1, 5)), - }, randomWords(randomInt(10, 100))); + [Action.CreateNoteAndResource]: async (context, user) => { + const resourceContent = randomWords(20); + const resourceId = randomHash(); - if (!context.createdResourceIds[sessionId]) context.createdResourceIds[sessionId] = []; - context.createdResourceIds[sessionId].push(item.jop_id); - - const noteItem = await createRandomNote(sessionId, { - body: `[](:/${item.jop_id})`, + const metadataBody = makeResourceSerializedBody({ + id: resourceId, + title: randomWords(5), + size: resourceContent.length, }); - if (!context.createdNoteIds[sessionId]) context.createdNoteIds[sessionId] = []; - context.createdNoteIds[sessionId].push(noteItem.jop_id); - }, - - [Action.UpdateNote]: async (context, sessionId) => { - const noteId = randomElement(context.createdNoteIds[sessionId]); - if (!noteId) return; - - await updateNote(sessionId, { - id: noteId, - title: randomWords(randomInt(1, 10)), + await models().item().saveFromRawContent(user, { + name: `${resourceId}.md`, + body: Buffer.from(metadataBody), }); - }, - [Action.UpdateFolder]: async (context, sessionId) => { - const folderId = randomElement(context.createdFolderIds[sessionId]); - if (!folderId) return; - - await updateFolder(sessionId, { - id: folderId, - title: randomWords(randomInt(1, 10)), + await models().item().saveFromRawContent(user, { + name: `.resource/${resourceId}`, + body: Buffer.from(resourceContent), }); + + if (!context.createdResourceIds[user.id]) context.createdResourceIds[user.id] = []; + context.createdResourceIds[user.id].push(resourceId); + + const noteItem = await createRandomNote(user, { + body: `[](:/${resourceId})`, + }); + + if (!context.createdNoteIds[user.id]) context.createdNoteIds[user.id] = []; + context.createdNoteIds[user.id].push(noteItem.jop_id); + + return true; }, - [Action.DeleteNote]: async (context, sessionId) => { - const noteId = randomElement(context.createdNoteIds[sessionId]); - if (!noteId) return; - removeElement(context.createdNoteIds[sessionId], noteId); - await deleteItem(sessionId, noteId); + [Action.UpdateNote]: async (context, user) => { + const noteId = randomElement(context.createdNoteIds[user.id]); + if (!noteId) return false; + + try { + const noteItem = await models().item().loadByJopId(user.id, noteId); + const note = await models().item().loadAsJoplinItem(noteItem.id); + const serialized = makeNoteSerializedBody({ + title: randomWords(10), + ...note, + }); + + await models().item().saveFromRawContent(user, { + name: `${note.id}.md`, + body: Buffer.from(serialized), + }); + } catch (error) { + if (error.code === CustomErrorCode.NotFound) return false; + throw error; + } + + return true; }, - [Action.DeleteFolder]: async (context, sessionId) => { - const folderId = randomElement(context.createdFolderIds[sessionId]); - if (!folderId) return; - removeElement(context.createdFolderIds[sessionId], folderId); - await deleteItem(sessionId, folderId); + [Action.UpdateFolder]: async (context, user) => { + const folderId = randomElement(context.createdFolderIds[user.id]); + if (!folderId) return false; + + try { + const folderItem = await models().item().loadByJopId(user.id, folderId); + const folder = await models().item().loadAsJoplinItem(folderItem.id); + const serialized = makeFolderSerializedBody({ + title: randomWords(5), + ...folder, + }); + + await models().item().saveFromRawContent(user, { + name: `${folder.id}.md`, + body: Buffer.from(serialized), + }); + } catch (error) { + if (error.code === CustomErrorCode.NotFound) return false; + throw error; + } + + return true; + }, + + [Action.DeleteNote]: async (context, user) => { + const noteId = randomElement(context.createdNoteIds[user.id]); + if (!noteId) return false; + const item = await models().item().loadByJopId(user.id, noteId, { fields: ['id'] }); + await models().item().delete(item.id, { allowNoOp: true }); + return true; + }, + + [Action.DeleteFolder]: async (context, user) => { + const folderId = randomElement(context.createdFolderIds[user.id]); + if (!folderId) return false; + const item = await models().item().loadByJopId(user.id, folderId, { fields: ['id'] }); + await models().item().delete(item.id, { allowNoOp: true }); + return true; }, }; const randomActionKey = () => { - return randomElement(Object.keys(reactions)) as Action; + const r = Math.random(); + if (r <= .5) { + return randomElement(createActions); + } else if (r <= .8) { + return randomElement(updateActions); + } else { + return randomElement(deleteActions); + } }; -const main = async (options?: Options) => { - options = { - userCount: 10, - minNoteCountPerUser: 0, - maxNoteCountPerUser: 1000, - minFolderCountPerUser: 0, - maxFolderCountPerUser: 50, - ...options, - }; +const main = async (_options?: Options) => { + // options = { + // userCount: 10, + // minNoteCountPerUser: 0, + // maxNoteCountPerUser: 1000, + // minFolderCountPerUser: 0, + // maxFolderCountPerUser: 50, + // ...options, + // }; shimInit({ nodeSqlite }); await beforeAllDb('populateDatabase'); logger().info(`Populating database: ${createdDbPath()}`); - const userAndSessions: UserAndSession[] = []; - - for (let i = 0; i < options.userCount; i++) { - logger().info(`Creating user ${i}`); - userAndSessions.push(await createUserAndSession(i, false)); - } - const context: Context = { createdNoteIds: {}, createdFolderIds: {}, @@ -183,39 +262,74 @@ const main = async (options?: Options) => { if (isDeleteAction(action)) report.deleted++; }; + let users: User[] = []; + + // ------------------------------------------------------------- + // CREATE USERS + // ------------------------------------------------------------- + { const promises = []; for (let i = 0; i < 1000; i++) { - const userAndSession = randomElement(userAndSessions); - const key = randomElement(createActions); - updateReport(key); promises.push((async () => { - await reactions[key](context, userAndSession.session.id); - logger().info(`Done action ${i}: ${key}. User: ${userAndSession.user.email}`); + const user = await models().user().save({ + full_name: `Toto ${i}`, + email: `toto${i}@example.com`, + password: '$2a$10$/2DMDnrx0PAspJ2DDnW/PO5x5M9H1abfSPsqxlPMhYiXgDi25751u', // Password = 111111 + }); + + users.push(user); + + logger().info(`Created user ${i}`); })()); } await Promise.all(promises); } + users = await models().user().loadByIds(users.map(u => u.id)); + + // ------------------------------------------------------------- + // CREATE NOTES, FOLDERS AND RESOURCES + // ------------------------------------------------------------- + { - let promises = []; + const promises = []; for (let i = 0; i < 1000; i++) { - const userAndSession = randomElement(userAndSessions); - const key = randomActionKey(); - updateReport(key); - promises.push((async () => { - await reactions[key](context, userAndSession.session.id); - logger().info(`Done action ${i}: ${key}. User: ${userAndSession.user.email}`); + const user = randomElement(users); + const action = randomElement(createActions); + await reactions[action](context, user); + updateReport(action); + logger().info(`Done action ${i}: ${action}. User: ${user.email}`); })()); + } - if (promises.length > 100) { - await Promise.all(promises); - promises = []; - } + await Promise.all(promises); + } + + // ------------------------------------------------------------- + // CREATE/UPDATE/DELETE NOTES, FOLDERS AND RESOURCES + // ------------------------------------------------------------- + + { + const promises = []; + + for (let i = 0; i < 10000; i++) { + promises.push((async () => { + const user = randomElement(users); + const action = randomActionKey(); + try { + const done = await reactions[action](context, user); + if (done) updateReport(action); + logger().info(`Done action ${i}: ${action}. User: ${user.email}${!done ? ' (Skipped)' : ''}`); + } catch (error) { + error.message = `Could not do action ${i}: ${action}. User: ${user.email}: ${error.message}`; + throw error; + } + })()); } await Promise.all(promises); diff --git a/packages/server/src/utils/testing/testUtils.ts b/packages/server/src/utils/testing/testUtils.ts index f47f35b5bb..a03f2a6167 100644 --- a/packages/server/src/utils/testing/testUtils.ts +++ b/packages/server/src/utils/testing/testUtils.ts @@ -570,7 +570,16 @@ type_: 2`; } export function makeResourceSerializedBody(resource: ResourceEntity = {}): string { - return `Test Resource + resource = { + id: randomHash(), + mime: 'plain/text', + file_extension: 'txt', + size: 0, + title: 'Test Resource', + ...resource, + }; + + return `${resource.title} id: ${resource.id} mime: ${resource.mime} diff --git a/yarn.lock b/yarn.lock index e6c84b97f7..d97832b9b7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5136,6 +5136,7 @@ __metadata: "@joplin/utils": ~2.13 "@koa/cors": 3.4.3 "@rmp135/sql-ts": 1.18.0 + "@types/bcryptjs": 2.4.5 "@types/formidable": 3.4.3 "@types/fs-extra": 11.0.2 "@types/jest": 29.5.4 @@ -7721,6 +7722,13 @@ __metadata: languageName: node linkType: hard +"@types/bcryptjs@npm:2.4.5": + version: 2.4.5 + resolution: "@types/bcryptjs@npm:2.4.5" + checksum: f721d72d8e1374ee2a342ce90cc902e2308cd059317af6e663d752537e704ea73bb119a2d34a6a68475f80abc1342635f48570119e0381f83a202724974f1e9f + languageName: node + linkType: hard + "@types/body-parser@npm:*": version: 1.19.2 resolution: "@types/body-parser@npm:1.19.2"