diff --git a/gulpfile.esm.js/plugins/drawio.js b/gulpfile.esm.js/plugins/drawio.js index dffd53bbf05c7fab6c284e4e898d3a6052693786..66f3f16d6868d1df6a26663b0d7197c8602457ba 100644 --- a/gulpfile.esm.js/plugins/drawio.js +++ b/gulpfile.esm.js/plugins/drawio.js @@ -1,9 +1,8 @@ -import { callbackify } from "util"; import exec from "../util/exec"; import fs from "fs"; import logger from "gulplog"; import { mkTempFile } from "../util/tempdir"; -import { obj } from "through2"; +import parallel from "../util/parallel"; import PluginError from "plugin-error"; import { withBinary } from "../util/optional"; @@ -14,41 +13,39 @@ const drawio = withBinary( "drawio", (DRAWIO_BINARY) => function drawio() { - return obj( - callbackify(async function (input) { - try { - const output = await mkTempFile(input); - output.extname = ".svg"; + return parallel(async function (input) { + try { + const output = await mkTempFile(input); + output.extname = ".svg"; - const args = [ - "--export", - "--format", - "svg", - "--width", - "1024", - "--output", - output.path, - input.path, - ]; + const args = [ + "--export", + "--format", + "svg", + "--width", + "1024", + "--output", + output.path, + input.path, + ]; - await exec(PLUGIN_NAME, DRAWIO_BINARY, args); + await exec(PLUGIN_NAME, DRAWIO_BINARY, args); - logger.info( - "%s: %s -> %s", - PLUGIN_NAME, - input.relative, - output.relative - ); + logger.info( + "%s: %s -> %s", + PLUGIN_NAME, + input.relative, + output.relative + ); - output.contents = fs.createReadStream(output.path, { - encoding: "UTF-8", - }); - this.push(output); - } catch (error) { - throw new PluginError(PLUGIN_NAME, error, { showStack: true }); - } - }) - ); + output.contents = fs.createReadStream(output.path, { + encoding: "UTF-8", + }); + this.push(output); + } catch (error) { + throw new PluginError(PLUGIN_NAME, error, { showStack: true }); + } + }); } ); diff --git a/gulpfile.esm.js/plugins/pandoc.js b/gulpfile.esm.js/plugins/pandoc.js index ae5e66a36ffbba828963017450e7503840760fd3..8fd6ed818fae79eac6aa0764d0cc7a5d501672dd 100644 --- a/gulpfile.esm.js/plugins/pandoc.js +++ b/gulpfile.esm.js/plugins/pandoc.js @@ -1,11 +1,11 @@ -import { callbackify, promisify } from "util"; import exec from "../util/exec"; import fs from "fs"; import { merge } from "merge-anything"; import { mkTempFile } from "../util/tempdir"; -import { obj } from "through2"; +import parallel from "../util/parallel"; import path from "path"; import PluginError from "plugin-error"; +import { promisify } from "util"; const writeFile = promisify(fs.writeFile); const exists = promisify(fs.exists); @@ -87,41 +87,39 @@ export default function pandoc(options = {}) { ); const revealJSURLResolver = getRevealJSResolver(revealJSURL); - return obj( - callbackify( - /** - * @param {import("vinyl")} input - */ - async function (input) { - const output = input.clone({ contents: false }); - output.extname = ".html"; - output.contents = null; + return parallel( + /** + * @param {import("vinyl")} input + */ + async function (input) { + const output = input.clone({ contents: false }); + output.extname = ".html"; + output.contents = null; - const src = await onDisk(input); + const src = await onDisk(input); - const args = [ - "--from=markdown+backtick_code_blocks+pandoc_title_block+yaml_metadata_block", - "--to=revealjs", - "--standalone", - `--template=${template}`, - `--slide-level=${slideLevel}`, - "--toc", - `--toc-depth=${tocDepth}`, - `--variable=revealjs-url:${revealJSURLResolver(output)}`, - ...Object.getOwnPropertyNames(variables).map( - (name) => `--variable=${name}:${variables[name]}` - ), - src.path, - ]; + const args = [ + "--from=markdown+backtick_code_blocks+pandoc_title_block+yaml_metadata_block", + "--to=revealjs", + "--standalone", + `--template=${template}`, + `--slide-level=${slideLevel}`, + "--toc", + `--toc-depth=${tocDepth}`, + `--variable=revealjs-url:${revealJSURLResolver(output)}`, + ...Object.getOwnPropertyNames(variables).map( + (name) => `--variable=${name}:${variables[name]}` + ), + src.path, + ]; - const { stdout } = await exec(PLUGIN_NAME, "pandoc", args, { - encoding: "buffer", - cwd: input.cwd, - }); - output.contents = stdout; + const { stdout } = await exec(PLUGIN_NAME, "pandoc", args, { + encoding: "buffer", + cwd: input.cwd, + }); + output.contents = stdout; - this.push(output); - } - ) + this.push(output); + } ); } diff --git a/gulpfile.esm.js/plugins/wkhtmltopdf.js b/gulpfile.esm.js/plugins/wkhtmltopdf.js index 6280709fc42dad316b26be4a86d3fb38a4991cf2..420fd8f5d9a7e4f19ba66651765491f4fa8c7c1c 100644 --- a/gulpfile.esm.js/plugins/wkhtmltopdf.js +++ b/gulpfile.esm.js/plugins/wkhtmltopdf.js @@ -1,8 +1,7 @@ -import { callbackify } from "util"; import { createReadStream } from "fs"; import exec from "../util/exec"; import { mkTempFile } from "../util/tempdir"; -import { obj } from "through2"; +import parallel from "../util/parallel"; import path from "path"; import PluginError from "plugin-error"; import { withBinary } from "../util/optional"; @@ -46,8 +45,7 @@ export const wkhtmltopdf = withBinary( (WKHTMLTOPDF_BINARY) => function wkhtmltopdf(options = {}) { const { args } = Object.assign({}, DEFAULT_OPTIONS, options); - return obj( - callbackify(async function (input) { + return parallel(async function (input) { try { const output = await mkTempFile(getPDFOutput(input)); const execArgs = [ @@ -64,7 +62,7 @@ export const wkhtmltopdf = withBinary( throw new PluginError(PLUGIN_NAME, err, { filename: input.path }); } }) - ); + ; } ); diff --git a/gulpfile.esm.js/util/parallel.js b/gulpfile.esm.js/util/parallel.js new file mode 100644 index 0000000000000000000000000000000000000000..e082f942b8c57f2dbd8b110dfae67f3017a7c3c0 --- /dev/null +++ b/gulpfile.esm.js/util/parallel.js @@ -0,0 +1,28 @@ + +import { cpus } from "os"; +import { obj } from "through2"; +import pMap from "p-map"; + +const concurrency = cpus().length; + +/** + * + * @param {(file) => Promise<any>} mapper + * @return {import('stream').Transform} + */ +export default function parallel(mapper) { + const inputs = []; + + return obj( + function collect(input, _enc, done) { + inputs.push(input); + done(); + }, + function mapAll(done) { + pMap(inputs, mapper.bind(this), { concurrency, stopOnError: false }).then( + () => done(), + error => done(error) + ); + } + ); +} diff --git a/package-lock.json b/package-lock.json index 29c4308ba9be930188b8fc697214a0de1da0643e..2e8758b49f77dff57eef365a4048e7479c376d66 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2916,6 +2916,16 @@ "p-map": "^3.0.0", "rimraf": "^3.0.0", "slash": "^3.0.0" + }, + "dependencies": { + "p-map": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-3.0.0.tgz", + "integrity": "sha512-d3qXVTF/s+W+CdJ5A29wywV2n8CQQYahlgz2bFiA+4eVNJbHJodPZ+/gXwPGh0bOqA+j8S+6+ckmvLGPk1QpxQ==", + "requires": { + "aggregate-error": "^3.0.0" + } + } } }, "delayed-stream": { @@ -8308,9 +8318,9 @@ } }, "p-map": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/p-map/-/p-map-3.0.0.tgz", - "integrity": "sha512-d3qXVTF/s+W+CdJ5A29wywV2n8CQQYahlgz2bFiA+4eVNJbHJodPZ+/gXwPGh0bOqA+j8S+6+ckmvLGPk1QpxQ==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz", + "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", "requires": { "aggregate-error": "^3.0.0" } diff --git a/package.json b/package.json index b7dfd65df71e60176352ec258e9dbba77e8be833..8b5c5a5f7303f5c3be0974b3187cea95f5ba8057 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "gulp-server-livereload": "^1.9.2", "gulplog": "^1.0.0", "merge-anything": "^3.0.3", + "p-map": "^4.0.0", "plugin-error": "^1.0.1", "through2": "^3.0.1", "vinyl": "^2.2.0",