Joplin
/
joplin
mirror of https://github.com/laurent22/joplin.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Tools: Various improvements on plugin-repo-cli

pull/4405/head
Laurent Cozic 2021-01-21 00:12:59 +00:00
parent f37d37e613
commit f9b0e2f6df
13 changed files with 539 additions and 3286 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.eslintignore
View File

@ -1374,12 +1374,18 @@ packages/lib/uuid.js.map
packages/lib/versionInfo.d.ts
packages/lib/versionInfo.js
packages/lib/versionInfo.js.map
packages/plugin-repo-cli/dummy.test.d.ts
packages/plugin-repo-cli/dummy.test.js
packages/plugin-repo-cli/dummy.test.js.map
packages/plugin-repo-cli/index.d.ts
packages/plugin-repo-cli/index.js
packages/plugin-repo-cli/index.js.map
packages/plugin-repo-cli/lib/checkIfPluginCanBeAdded.d.ts
packages/plugin-repo-cli/lib/checkIfPluginCanBeAdded.js
packages/plugin-repo-cli/lib/checkIfPluginCanBeAdded.js.map
packages/plugin-repo-cli/lib/checkIfPluginCanBeAdded.test.d.ts
packages/plugin-repo-cli/lib/checkIfPluginCanBeAdded.test.js
packages/plugin-repo-cli/lib/checkIfPluginCanBeAdded.test.js.map
packages/plugin-repo-cli/lib/updateReadme.d.ts
packages/plugin-repo-cli/lib/updateReadme.js
packages/plugin-repo-cli/lib/updateReadme.js.map
packages/plugins/ToggleSidebars/api/index.d.ts
packages/plugins/ToggleSidebars/api/index.js
packages/plugins/ToggleSidebars/api/index.js.map

12
.gitignore vendored
View File

@ -1362,12 +1362,18 @@ packages/lib/uuid.js.map
packages/lib/versionInfo.d.ts
packages/lib/versionInfo.js
packages/lib/versionInfo.js.map
packages/plugin-repo-cli/dummy.test.d.ts
packages/plugin-repo-cli/dummy.test.js
packages/plugin-repo-cli/dummy.test.js.map
packages/plugin-repo-cli/index.d.ts
packages/plugin-repo-cli/index.js
packages/plugin-repo-cli/index.js.map
packages/plugin-repo-cli/lib/checkIfPluginCanBeAdded.d.ts
packages/plugin-repo-cli/lib/checkIfPluginCanBeAdded.js
packages/plugin-repo-cli/lib/checkIfPluginCanBeAdded.js.map
packages/plugin-repo-cli/lib/checkIfPluginCanBeAdded.test.d.ts
packages/plugin-repo-cli/lib/checkIfPluginCanBeAdded.test.js
packages/plugin-repo-cli/lib/checkIfPluginCanBeAdded.test.js.map
packages/plugin-repo-cli/lib/updateReadme.d.ts
packages/plugin-repo-cli/lib/updateReadme.js
packages/plugin-repo-cli/lib/updateReadme.js.map
packages/plugins/ToggleSidebars/api/index.d.ts
packages/plugins/ToggleSidebars/api/index.js
packages/plugins/ToggleSidebars/api/index.js.map

16
packages/lib/services/plugins/utils/validatePluginId.test.ts
View File

@ -4,11 +4,12 @@ describe('validatePluginId', () => {
test('should validate an ID', () => {
const okCases = [
'thatsok',
'that-s-ok',
'that_s_fine12',
'com.ok.too',
'Good',
'joplinapp.org.plugins.thatsok',
'joplinapp.org.plugins.that-s-ok',
'joplinapp.org.plugins.that_s_fine12',
'joplinapp.org.plugins.com.ok.too',
'joplinapp.org.plugins.Good',
'outline',
];
const errorCases = [
@ -16,8 +17,9 @@ describe('validatePluginId', () => {
'verylongidverylongidverylongidverylongidverylongidverylongidverylongidverylongidverylongidverylongidverylongidverylongidverylongidverylongidverylongidverylongidverylongidverylongidverylongidverylongidverylongidverylongidverylongidverylongidverylongidverylongidverylongidverylongidverylongid',
'-shouldstartwiththis',
'shouldntendwithit.',
' no space ',
'no space',
' no space no space no space no space ',
'no spaceno spaceno spaceno spaceno space',
'tooshort',
];
for (const t of okCases) {

2
packages/lib/services/plugins/utils/validatePluginId.ts
View File

@ -1,5 +1,7 @@
export default function(id: string): void {
if (!id) throw new Error('ID cannot be empty');
// Allow a few exceptions that existed before the limit was added
if (id.length < 16 && !['Outline', 'outline', 'MyPlugin'].includes(id)) throw new Error('ID cannot be shorter than 16 characters');
if (id.length > 256) throw new Error('ID cannot be longer than 256 characters');
const whitelist = '0-9a-zA-Z._-';

9
packages/plugin-repo-cli/dummy.test.ts
View File

@ -1,9 +0,0 @@
// Dummy test because the Jest setup is done but there's for now no test.
describe('dummy', () => {
it('should pass', () => {
expect(1).toBe(1);
});
});

183
packages/plugin-repo-cli/index.ts
View File

@ -4,8 +4,9 @@ import * as fs from 'fs-extra';
import * as path from 'path';
import * as process from 'process';
import validatePluginId from '@joplin/lib/services/plugins/utils/validatePluginId';
import markdownUtils, { MarkdownTableHeader, MarkdownTableRow } from '@joplin/lib/markdownUtils';
import { execCommand2, resolveRelativePathWithinDir, gitPullTry, gitRepoCleanTry, gitRepoClean } from '@joplin/tools/tool-utils.js';
import checkIfPluginCanBeAdded from './lib/checkIfPluginCanBeAdded';
import updateReadme from './lib/updateReadme';
interface NpmPackage {
name: string;
@ -45,11 +46,10 @@ async function checkPluginRepository(dirPath: string) {
if (!(await fs.pathExists(dirPath))) throw new Error(`No plugin repository at: ${dirPath}`);
if (!(await fs.pathExists(`${dirPath}/.git`))) throw new Error(`Directory is not a Git repository: ${dirPath}`);
const previousDir = process.cwd();
process.chdir(dirPath);
const previousDir = chdir(dirPath);
await gitRepoCleanTry();
await gitPullTry();
process.chdir(previousDir);
chdir(previousDir);
}
async function readJsonFile(manifestPath: string, defaultValue: any = null): Promise<any> {
@ -62,16 +62,8 @@ async function readJsonFile(manifestPath: string, defaultValue: any = null): Pro
return JSON.parse(content);
}
function caseInsensitiveFindManifest(manifests: any, manifestId: string): any {
for (const id of Object.keys(manifests)) {
if (id.toLowerCase() === manifestId.toLowerCase()) return manifests[id];
}
return null;
}
async function extractPluginFilesFromPackage(existingManifests: any, workDir: string, packageName: string, destDir: string): Promise<any> {
const previousDir = process.cwd();
process.chdir(workDir);
const previousDir = chdir(workDir);
await execCommand2(`npm install ${packageName} --save --ignore-scripts`, { showOutput: false });
@ -93,18 +85,7 @@ async function extractPluginFilesFromPackage(existingManifests: any, workDir: st
manifest._npm_package_name = packageName;
// If there's already a plugin with this ID published under a different
// package name, we skip it. Otherwise it would allow anyone to overwrite
// someone else plugin just by using the same ID. So the first plugin with
// this ID that was originally added is kept.
//
// We need case insensitive match because the filesystem might be case
// insensitive too.
const originalManifest = caseInsensitiveFindManifest(existingManifests, manifest.id);
if (originalManifest && originalManifest._npm_package_name !== packageName) {
throw new Error(`Plugin "${manifest.id}" from npm package "${packageName}" has already been published under npm package "${originalManifest._npm_package_name}". Plugin from package "${packageName}" will not be imported.`);
}
checkIfPluginCanBeAdded(existingManifests, manifest);
const pluginDestDir = resolveRelativePathWithinDir(destDir, manifest.id);
await fs.mkdirp(pluginDestDir);
@ -112,58 +93,11 @@ async function extractPluginFilesFromPackage(existingManifests: any, workDir: st
await fs.writeFile(path.resolve(pluginDestDir, 'manifest.json'), JSON.stringify(manifest, null, '\t'), 'utf8');
await fs.copy(pluginFilePath, path.resolve(pluginDestDir, 'plugin.jpl'));
process.chdir(previousDir);
chdir(previousDir);
return manifest;
}
async function updateReadme(readmePath: string, manifests: any) {
const rows: MarkdownTableRow[] = [];
for (const pluginId in manifests) {
rows.push(manifests[pluginId]);
}
const headers: MarkdownTableHeader[] = [
{
name: 'homepage_url',
label: '&nbsp;',
filter: (value: string) => {
return `[🏠](${markdownUtils.escapeLinkUrl(value)})`;
},
},
{
name: 'name',
label: 'Name',
},
{
name: 'version',
label: 'Version',
},
{
name: 'description',
label: 'Description',
},
{
name: 'author',
label: 'Author',
},
];
rows.sort((a: any, b: any) => {
return a.name.toLowerCase() < b.name.toLowerCase() ? -1 : +1;
});
const mdTable = markdownUtils.createMarkdownTable(headers, rows);
const tableRegex = /<!-- PLUGIN_LIST -->([^]*)<!-- PLUGIN_LIST -->/;
const content = await fs.pathExists(readmePath) ? await fs.readFile(readmePath, 'utf8') : '<!-- PLUGIN_LIST -->\n<!-- PLUGIN_LIST -->';
const newContent = content.replace(tableRegex, `<!-- PLUGIN_LIST -->\n${mdTable}\n<!-- PLUGIN_LIST -->`);
await fs.writeFile(readmePath, newContent, 'utf8');
}
interface CommandBuildArgs {
pluginRepoDir: string;
}
@ -173,29 +107,54 @@ enum ProcessingActionType {
Update = 2,
}
function commitMessage(actionType: ProcessingActionType, npmPackage: NpmPackage): string {
function commitMessage(actionType: ProcessingActionType, manifest: any, npmPackage: NpmPackage, error: any): string {
const output: string[] = [];
if (actionType === ProcessingActionType.Add) {
output.push('New');
} else {
output.push('Update');
}
if (!error) {
if (actionType === ProcessingActionType.Add) {
output.push('New');
} else {
output.push('Update');
}
output.push(`${npmPackage.name}@${npmPackage.version}`);
output.push(`${manifest.id}@${manifest.version}`);
} else {
output.push(`Error: ${npmPackage.name}@${npmPackage.version}`);
}
return output.join(': ');
}
function pluginManifestsPath(repoDir: string): string {
return path.resolve(repoDir, 'manifests.json');
}
async function readManifests(repoDir: string): Promise<any> {
return readJsonFile(pluginManifestsPath(repoDir), {});
}
async function writeManifests(repoDir: string, manifests: any) {
await fs.writeFile(pluginManifestsPath(repoDir), JSON.stringify(manifests, null, '\t'), 'utf8');
}
function chdir(path: string): string {
const previous = process.cwd();
try {
process.chdir(path);
} catch (error) {
throw new Error(`Could not chdir to path: ${path}`);
}
return previous;
}
async function processNpmPackage(npmPackage: NpmPackage, repoDir: string) {
const tempDir = `${repoDir}/temp`;
const pluginManifestsPath = path.resolve(repoDir, 'manifests.json');
const obsoleteManifestsPath = path.resolve(repoDir, 'obsoletes.json');
const errorsPath = path.resolve(repoDir, 'errors.json');
await fs.mkdirp(tempDir);
const originalPluginManifests = await readJsonFile(pluginManifestsPath, {});
const originalPluginManifests = await readManifests(repoDir);
const obsoleteManifests = await readJsonFile(obsoleteManifestsPath, {});
const existingManifests = {
...originalPluginManifests,
@ -205,8 +164,7 @@ async function processNpmPackage(npmPackage: NpmPackage, repoDir: string) {
const packageTempDir = `${tempDir}/packages`;
await fs.mkdirp(packageTempDir);
const previousDir = process.cwd();
process.chdir(packageTempDir);
chdir(packageTempDir);
await execCommand2('npm init --yes --loglevel silent', { quiet: true });
const errors: any = await readJsonFile(errorsPath, {});
@ -214,48 +172,50 @@ async function processNpmPackage(npmPackage: NpmPackage, repoDir: string) {
let actionType: ProcessingActionType = ProcessingActionType.Update;
let manifests: any = {};
let manifest: any = {};
let error: any = null;
try {
const destDir = `${repoDir}/plugins/`;
const manifest = await extractPluginFilesFromPackage(existingManifests, packageTempDir, npmPackage.name, destDir);
manifest = await extractPluginFilesFromPackage(existingManifests, packageTempDir, npmPackage.name, destDir);
if (!existingManifests[manifest.id]) {
actionType = ProcessingActionType.Add;
}
if (!obsoleteManifests[manifest.id]) manifests[manifest.id] = manifest;
} catch (error) {
console.error(error);
errors[npmPackage.name] = error.message || '';
} catch (e) {
console.error(e);
errors[npmPackage.name] = e.message || '';
error = e;
}
// We preserve the original manifests so that if a plugin has been removed
// from npm, we still keep it. It's also a security feature - it means that
// if a plugin is removed from npm, it's not possible to highjack it by
// creating a new npm package with the same plugin ID.
manifests = {
...originalPluginManifests,
...manifests,
};
await fs.writeFile(pluginManifestsPath, JSON.stringify(manifests, null, '\t'), 'utf8');
if (Object.keys(errors).length) {
await fs.writeFile(errorsPath, JSON.stringify(errors, null, '\t'), 'utf8');
} else {
await fs.remove(errorsPath);
}
await updateReadme(`${repoDir}/README.md`, manifests);
if (!error) {
// We preserve the original manifests so that if a plugin has been removed
// from npm, we still keep it. It's also a security feature - it means that
// if a plugin is removed from npm, it's not possible to highjack it by
// creating a new npm package with the same plugin ID.
manifests = {
...originalPluginManifests,
...manifests,
};
process.chdir(previousDir);
await writeManifests(repoDir, manifests);
await updateReadme(`${repoDir}/README.md`, manifests);
}
chdir(repoDir);
await fs.remove(tempDir);
process.chdir(repoDir);
if (!(await gitRepoClean())) {
await execCommand2('git add -A', { showOutput: false });
await execCommand2(['git', 'commit', '-m', commitMessage(actionType, npmPackage)], { showOutput: false });
await execCommand2(['git', 'commit', '-m', commitMessage(actionType, manifest, npmPackage, error)], { showOutput: false });
} else {
console.info('Nothing to commit');
}
@ -267,6 +227,21 @@ async function commandBuild(args: CommandBuildArgs) {
const repoDir = args.pluginRepoDir;
await checkPluginRepository(repoDir);
// When starting, always update and commit README, in case something has
// been updated via a pull request (for example obsoletes.json being
// modified). We do that separately so that the README update doesn't get
// mixed up with plugin updates, as in this example:
// https://github.com/joplin/plugins/commit/8a65bbbf64bf267674f854a172466ffd4f07c672
const manifests = await readManifests(repoDir);
await updateReadme(`${repoDir}/README.md`, manifests);
const previousDir = chdir(repoDir);
if (!(await gitRepoClean())) {
console.info('Updating README...');
await execCommand2('git add -A', { showOutput: true });
await execCommand2('git commit -m "Update README"', { showOutput: true });
}
chdir(previousDir);
const searchResults = (await execCommand2('npm search joplin-plugin --searchlimit 5000 --json', { showOutput: false })).trim();
const npmPackages = pluginInfoFromSearchResults(JSON.parse(searchResults));

15
packages/plugin-repo-cli/jest.config.js Normal file
View File

@ -0,0 +1,15 @@
module.exports = {
testMatch: [
'**/*.test.js',
],
testPathIgnorePatterns: [
'<rootDir>/node_modules/',
],
testEnvironment: 'node',
slowTestThreshold: 30,
// setupFilesAfterEnv: [`${__dirname}/jest.setup.js`],
};

93
packages/plugin-repo-cli/lib/checkIfPluginCanBeAdded.test.ts Normal file
View File

@ -0,0 +1,93 @@
import checkIfPluginCanBeAdded from './checkIfPluginCanBeAdded';
describe('checkIfPluginCanBeAdded', () => {
test('should not add if already a plugin with this ID but different package name', () => {
const testCases = [
[
{
'test': {
id: 'test',
_npm_package_name: 'original',
},
},
{
id: 'test',
_npm_package_name: 'original',
},
true,
],
[
{
'test': {
id: 'test',
_npm_package_name: 'original',
},
},
{
id: 'test',
_npm_package_name: 'cantdothat',
},
false,
],
];
for (const t of testCases) {
const [existingManifests, manifest, shouldWork] = t;
let hasThrown = false;
try {
checkIfPluginCanBeAdded(existingManifests, manifest);
} catch (error) {
hasThrown = true;
}
expect(!hasThrown).toBe(shouldWork);
}
});
test('should not add if already a plugin with this ID but a different case', () => {
const testCases = [
[
{
'test': {
id: 'test',
_npm_package_name: 'original',
},
},
{
id: 'newone',
_npm_package_name: 'otherpackage',
},
true,
],
[
{
'test': {
id: 'test',
_npm_package_name: 'original',
},
},
{
id: 'Test',
_npm_package_name: 'original',
},
false,
],
];
for (const t of testCases) {
const [existingManifests, manifest, shouldWork] = t;
let hasThrown = false;
try {
checkIfPluginCanBeAdded(existingManifests, manifest);
} catch (error) {
hasThrown = true;
}
expect(!hasThrown).toBe(shouldWork);
}
});
});

27
packages/plugin-repo-cli/lib/checkIfPluginCanBeAdded.ts Normal file
View File

@ -0,0 +1,27 @@
function caseInsensitiveFindManifest(manifests: any, manifestId: string): any {
for (const id of Object.keys(manifests)) {
if (id.toLowerCase() === manifestId.toLowerCase()) return manifests[id];
}
return null;
}
export default function(existingManifests: any, manifest: any) {
// If there's already a plugin with this ID published under a different
// package name, we skip it. Otherwise it would allow anyone to overwrite
// someone else plugin just by using the same ID. So the first plugin with
// this ID that was originally added is kept.
//
// We need case insensitive match because the filesystem might be case
// insensitive too.
const originalManifest = caseInsensitiveFindManifest(existingManifests, manifest.id);
if (originalManifest && originalManifest._npm_package_name !== manifest._npm_package_name) {
throw new Error(`Plugin "${manifest.id}" from npm package "${manifest._npm_package_name}" has already been published under npm package "${originalManifest._npm_package_name}". Plugin from package "${originalManifest._npm_package_name}" will not be imported.`);
}
// Don't add a plugin if there is already a plugin with the same ID but
// different casing.
if (originalManifest && originalManifest.id !== manifest.id) {
throw new Error(`Plugin "${manifest.id}" cannot be published because there is already a plugin with ID "${originalManifest.id}". A new package, under a different name, should be published if the plugin ID needs to change.`);
}
}

49
packages/plugin-repo-cli/lib/updateReadme.ts Normal file
View File

@ -0,0 +1,49 @@
import * as fs from 'fs-extra';
import markdownUtils, { MarkdownTableHeader, MarkdownTableRow } from '@joplin/lib/markdownUtils';
export default async function(readmePath: string, manifests: any) {
const rows: MarkdownTableRow[] = [];
for (const pluginId in manifests) {
rows.push(manifests[pluginId]);
}
const headers: MarkdownTableHeader[] = [
{
name: 'homepage_url',
label: '&nbsp;',
filter: (value: string) => {
return `[🏠](${markdownUtils.escapeLinkUrl(value)})`;
},
},
{
name: 'name',
label: 'Name',
},
{
name: 'version',
label: 'Version',
},
{
name: 'description',
label: 'Description',
},
{
name: 'author',
label: 'Author',
},
];
rows.sort((a: any, b: any) => {
return a.name.toLowerCase() < b.name.toLowerCase() ? -1 : +1;
});
const mdTable = markdownUtils.createMarkdownTable(headers, rows);
const tableRegex = /<!-- PLUGIN_LIST -->([^]*)<!-- PLUGIN_LIST -->/;
const content = await fs.pathExists(readmePath) ? await fs.readFile(readmePath, 'utf8') : '<!-- PLUGIN_LIST -->\n<!-- PLUGIN_LIST -->';
const newContent = content.replace(tableRegex, `<!-- PLUGIN_LIST -->\n${mdTable}\n<!-- PLUGIN_LIST -->`);
await fs.writeFile(readmePath, newContent, 'utf8');
}

3391
packages/plugin-repo-cli/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

15
packages/tools/package-lock.json generated
View File

@ -657,8 +657,7 @@
"buffer-from": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz",
"integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==",
"dev": true
"integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A=="
},
"cache-base": {
"version": "1.0.1",
@ -4032,8 +4031,7 @@
"source-map": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
"integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
"dev": true
"integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
},
"source-map-resolve": {
"version": "0.5.3",
@ -4048,6 +4046,15 @@
"urix": "^0.1.0"
}
},
"source-map-support": {
"version": "0.5.19",
"resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz",
"integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==",
"requires": {
"buffer-from": "^1.0.0",
"source-map": "^0.6.0"
}
},
"source-map-url": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/source-map-url/-/source-map-url-0.4.0.tgz",

1
packages/tools/package.json
View File

@ -31,6 +31,7 @@
"relative": "^3.0.2",
"request": "^2.88.0",
"sharp": "^0.25.2",
"source-map-support": "^0.5.19",
"uri-template": "^1.0.1",
"yargs": "^16.0.3"
},