From a8cfed2170e850c068194f2f665c8ec5aefcb071 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillaume=20Perr=C3=A9al?= <guillaume.perreal@inrae.fr> Date: Wed, 3 Jun 2020 16:07:33 +0200 Subject: [PATCH] =?UTF-8?q?Met=20en=20place=20une=20ex=C3=A9cution=20paral?= =?UTF-8?q?l=C3=A8le=20pour=20les=20filtres=20drawio,=20pandoc=20et=20wkht?= =?UTF-8?q?mltopdf.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- gulpfile.esm.js/plugins/drawio.js | 63 ++++++++++++------------ gulpfile.esm.js/plugins/pandoc.js | 66 +++++++++++++------------- gulpfile.esm.js/plugins/wkhtmltopdf.js | 8 ++-- gulpfile.esm.js/util/parallel.js | 28 +++++++++++ package-lock.json | 16 +++++-- package.json | 1 + 6 files changed, 107 insertions(+), 75 deletions(-) create mode 100644 gulpfile.esm.js/util/parallel.js diff --git a/gulpfile.esm.js/plugins/drawio.js b/gulpfile.esm.js/plugins/drawio.js index dffd53b..66f3f16 100644 --- a/gulpfile.esm.js/plugins/drawio.js +++ b/gulpfile.esm.js/plugins/drawio.js @@ -1,9 +1,8 @@ -import { callbackify } from "util"; import exec from "../util/exec"; import fs from "fs"; import logger from "gulplog"; import { mkTempFile } from "../util/tempdir"; -import { obj } from "through2"; +import parallel from "../util/parallel"; import PluginError from "plugin-error"; import { withBinary } from "../util/optional"; @@ -14,41 +13,39 @@ const drawio = withBinary( "drawio", (DRAWIO_BINARY) => function drawio() { - return obj( - callbackify(async function (input) { - try { - const output = await mkTempFile(input); - output.extname = ".svg"; + return parallel(async function (input) { + try { + const output = await mkTempFile(input); + output.extname = ".svg"; - const args = [ - "--export", - "--format", - "svg", - "--width", - "1024", - "--output", - output.path, - input.path, - ]; + const args = [ + "--export", + "--format", + "svg", + "--width", + "1024", + "--output", + output.path, + input.path, + ]; - await exec(PLUGIN_NAME, DRAWIO_BINARY, args); + await exec(PLUGIN_NAME, DRAWIO_BINARY, args); - logger.info( - "%s: %s -> %s", - PLUGIN_NAME, - input.relative, - output.relative - ); + logger.info( + "%s: %s -> %s", + PLUGIN_NAME, + input.relative, + output.relative + ); - output.contents = fs.createReadStream(output.path, { - encoding: "UTF-8", - }); - this.push(output); - } catch (error) { - throw new PluginError(PLUGIN_NAME, error, { showStack: true }); - } - }) - ); + output.contents = fs.createReadStream(output.path, { + encoding: "UTF-8", + }); + this.push(output); + } catch (error) { + throw new PluginError(PLUGIN_NAME, error, { showStack: true }); + } + }); } ); diff --git a/gulpfile.esm.js/plugins/pandoc.js b/gulpfile.esm.js/plugins/pandoc.js index ae5e66a..8fd6ed8 100644 --- a/gulpfile.esm.js/plugins/pandoc.js +++ b/gulpfile.esm.js/plugins/pandoc.js @@ -1,11 +1,11 @@ -import { callbackify, promisify } from "util"; import exec from "../util/exec"; import fs from "fs"; import { merge } from "merge-anything"; import { mkTempFile } from "../util/tempdir"; -import { obj } from "through2"; +import parallel from "../util/parallel"; import path from "path"; import PluginError from "plugin-error"; +import { promisify } from "util"; const writeFile = promisify(fs.writeFile); const exists = promisify(fs.exists); @@ -87,41 +87,39 @@ export default function pandoc(options = {}) { ); const revealJSURLResolver = getRevealJSResolver(revealJSURL); - return obj( - callbackify( - /** - * @param {import("vinyl")} input - */ - async function (input) { - const output = input.clone({ contents: false }); - output.extname = ".html"; - output.contents = null; + return parallel( + /** + * @param {import("vinyl")} input + */ + async function (input) { + const output = input.clone({ contents: false }); + output.extname = ".html"; + output.contents = null; - const src = await onDisk(input); + const src = await onDisk(input); - const args = [ - "--from=markdown+backtick_code_blocks+pandoc_title_block+yaml_metadata_block", - "--to=revealjs", - "--standalone", - `--template=${template}`, - `--slide-level=${slideLevel}`, - "--toc", - `--toc-depth=${tocDepth}`, - `--variable=revealjs-url:${revealJSURLResolver(output)}`, - ...Object.getOwnPropertyNames(variables).map( - (name) => `--variable=${name}:${variables[name]}` - ), - src.path, - ]; + const args = [ + "--from=markdown+backtick_code_blocks+pandoc_title_block+yaml_metadata_block", + "--to=revealjs", + "--standalone", + `--template=${template}`, + `--slide-level=${slideLevel}`, + "--toc", + `--toc-depth=${tocDepth}`, + `--variable=revealjs-url:${revealJSURLResolver(output)}`, + ...Object.getOwnPropertyNames(variables).map( + (name) => `--variable=${name}:${variables[name]}` + ), + src.path, + ]; - const { stdout } = await exec(PLUGIN_NAME, "pandoc", args, { - encoding: "buffer", - cwd: input.cwd, - }); - output.contents = stdout; + const { stdout } = await exec(PLUGIN_NAME, "pandoc", args, { + encoding: "buffer", + cwd: input.cwd, + }); + output.contents = stdout; - this.push(output); - } - ) + this.push(output); + } ); } diff --git a/gulpfile.esm.js/plugins/wkhtmltopdf.js b/gulpfile.esm.js/plugins/wkhtmltopdf.js index 6280709..420fd8f 100644 --- a/gulpfile.esm.js/plugins/wkhtmltopdf.js +++ b/gulpfile.esm.js/plugins/wkhtmltopdf.js @@ -1,8 +1,7 @@ -import { callbackify } from "util"; import { createReadStream } from "fs"; import exec from "../util/exec"; import { mkTempFile } from "../util/tempdir"; -import { obj } from "through2"; +import parallel from "../util/parallel"; import path from "path"; import PluginError from "plugin-error"; import { withBinary } from "../util/optional"; @@ -46,8 +45,7 @@ export const wkhtmltopdf = withBinary( (WKHTMLTOPDF_BINARY) => function wkhtmltopdf(options = {}) { const { args } = Object.assign({}, DEFAULT_OPTIONS, options); - return obj( - callbackify(async function (input) { + return parallel(async function (input) { try { const output = await mkTempFile(getPDFOutput(input)); const execArgs = [ @@ -64,7 +62,7 @@ export const wkhtmltopdf = withBinary( throw new PluginError(PLUGIN_NAME, err, { filename: input.path }); } }) - ); + ; } ); diff --git a/gulpfile.esm.js/util/parallel.js b/gulpfile.esm.js/util/parallel.js new file mode 100644 index 0000000..e082f94 --- /dev/null +++ b/gulpfile.esm.js/util/parallel.js @@ -0,0 +1,28 @@ + +import { cpus } from "os"; +import { obj } from "through2"; +import pMap from "p-map"; + +const concurrency = cpus().length; + +/** + * + * @param {(file) => Promise<any>} mapper + * @return {import('stream').Transform} + */ +export default function parallel(mapper) { + const inputs = []; + + return obj( + function collect(input, _enc, done) { + inputs.push(input); + done(); + }, + function mapAll(done) { + pMap(inputs, mapper.bind(this), { concurrency, stopOnError: false }).then( + () => done(), + error => done(error) + ); + } + ); +} diff --git a/package-lock.json b/package-lock.json index 29c4308..2e8758b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2916,6 +2916,16 @@ "p-map": "^3.0.0", "rimraf": "^3.0.0", "slash": "^3.0.0" + }, + "dependencies": { + "p-map": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-3.0.0.tgz", + "integrity": "sha512-d3qXVTF/s+W+CdJ5A29wywV2n8CQQYahlgz2bFiA+4eVNJbHJodPZ+/gXwPGh0bOqA+j8S+6+ckmvLGPk1QpxQ==", + "requires": { + "aggregate-error": "^3.0.0" + } + } } }, "delayed-stream": { @@ -8308,9 +8318,9 @@ } }, "p-map": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/p-map/-/p-map-3.0.0.tgz", - "integrity": "sha512-d3qXVTF/s+W+CdJ5A29wywV2n8CQQYahlgz2bFiA+4eVNJbHJodPZ+/gXwPGh0bOqA+j8S+6+ckmvLGPk1QpxQ==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz", + "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", "requires": { "aggregate-error": "^3.0.0" } diff --git a/package.json b/package.json index b7dfd65..8b5c5a5 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "gulp-server-livereload": "^1.9.2", "gulplog": "^1.0.0", "merge-anything": "^3.0.3", + "p-map": "^4.0.0", "plugin-error": "^1.0.1", "through2": "^3.0.1", "vinyl": "^2.2.0", -- GitLab