Use POSIX jobserver when available or -j<number> to run PDF
builds in parallel, restoring pdf build performance. Yet,
running it when debugging troubles is a bad idea, so, when
calling directly via command line, except if "-j" is splicitly
requested, it will serialize the build.
With such change, a PDF doc builds now takes around 5 minutes
on a Ryzen 9 machine with 32 cpu threads:
# Explicitly paralelize both Sphinx and LaTeX pdf builds
$ make cleandocs; time scripts/sphinx-build-wrapper pdfdocs -j 33
real 5m17.901s
user 15m1.499s
sys 2m31.482s
# Use POSIX jobserver to paralelize both sphinx-build and LaTeX
$ make cleandocs; time make pdfdocs
real 5m22.369s
user 15m9.076s
sys 2m31.419s
# Serializes PDF build, while keeping Sphinx parallelized.
# it is equivalent of passing -jauto via command line
$ make cleandocs; time scripts/sphinx-build-wrapper pdfdocs
real 11m20.901s
user 13m2.910s
sys 1m44.553s
Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@xxxxxxxxxx>
---
tools/docs/sphinx-build-wrapper | 158 +++++++++++++++++++++++---------
1 file changed, 117 insertions(+), 41 deletions(-)
diff --git a/tools/docs/sphinx-build-wrapper b/tools/docs/sphinx-build-wrapper
index c318af5f2af1..c5fcc448a275 100755
--- a/tools/docs/sphinx-build-wrapper
+++ b/tools/docs/sphinx-build-wrapper
@@ -52,6 +52,8 @@ import shutil
import subprocess
import sys
+from concurrent import futures
+
from lib.python_version import PythonVersion
LIB_DIR = "../../scripts/lib"
@@ -282,6 +284,82 @@ class SphinxBuilder:
except (OSError, IOError) as e:
print(f"Warning: Failed to copy CSS: {e}", file=sys.stderr)
+ def build_pdf_file(self, latex_cmd, from_dir, path):
+ """Builds a single pdf file using latex_cmd"""
+ try:
+ subprocess.run(latex_cmd + [path],
+ cwd=from_dir, check=True)
+
+ return True
+ except subprocess.CalledProcessError:
+ return False
+
+ def pdf_parallel_build(self, tex_suffix, latex_cmd, tex_files, n_jobs):
+ """Build PDF files in parallel if possible"""
+ builds = {}
+ build_failed = False
+ max_len = 0
+ has_tex = False
+
+ #
+ # LaTeX PDF error code is almost useless for us:
+ # any warning makes it non-zero. For kernel doc builds it always return
+ # non-zero even when build succeeds. So, let's do the best next thing:
+ # Ignore build errors. At the end, check if all PDF files were built,
+ # printing a summary with the built ones and returning 0 if all of
+ # them were actually built.
+ #
+ with futures.ThreadPoolExecutor(max_workers=n_jobs) as executor:
+ jobs = {}
+
+ for from_dir, pdf_dir, entry in tex_files:
+ name = entry.name
+
+ if not name.endswith(tex_suffix):
+ continue
+
+ name = name[:-len(tex_suffix)]
+
+ max_len = max(max_len, len(name))
+
+ has_tex = True
+
+ future = executor.submit(self.build_pdf_file, latex_cmd,
+ from_dir, entry.path)
+ jobs[future] = (from_dir, pdf_dir, name)
+
+ for future in futures.as_completed(jobs):
+ from_dir, pdf_dir, name = jobs[future]
+
+ pdf_name = name + ".pdf"
+ pdf_from = os.path.join(from_dir, pdf_name)
+
+ try:
+ success = future.result()
+
+ if success and os.path.exists(pdf_from):
+ pdf_to = os.path.join(pdf_dir, pdf_name)
+
+ os.rename(pdf_from, pdf_to)
+ builds[name] = os.path.relpath(pdf_to, self.builddir)
+ else:
+ builds[name] = "FAILED"
+ build_failed = True
+ except futures.Error as e:
+ builds[name] = f"FAILED ({repr(e)})"
+ build_failed = True
+
+ #
+ # Handle case where no .tex files were found
+ #
+ if not has_tex:
+ name = "Sphinx LaTeX builder"
+ max_len = max(max_len, len(name))
+ builds[name] = "FAILED (no .tex file was generated)"
+ build_failed = True
+
+ return builds, build_failed, max_len
+
def handle_pdf(self, output_dirs):
"""
Extra steps for PDF output.
@@ -292,7 +370,9 @@ class SphinxBuilder:
"""
builds = {}
max_len = 0
+ tex_suffix = ".tex"
+ tex_files = []
for from_dir in output_dirs:
pdf_dir = os.path.join(from_dir, "../pdf")
os.makedirs(pdf_dir, exist_ok=True)
@@ -304,55 +384,51 @@ class SphinxBuilder:
latex_cmd.extend(shlex.split(self.latexopts))
- tex_suffix = ".tex"
-
- #
- # Process each .tex file
- #
-
- has_tex = False
- build_failed = False
+ # Get a list of tex files to process
with os.scandir(from_dir) as it:
for entry in it:
- if not entry.name.endswith(tex_suffix):
- continue
+ if entry.name.endswith(tex_suffix):
+ tex_files.append((from_dir, pdf_dir, entry))
- name = entry.name[:-len(tex_suffix)]
- has_tex = True
+ #
+ # When using make, this won't be used, as the number of jobs comes
+ # from POSIX jobserver. So, this covers the case where build comes
+ # from command line. On such case, serialize by default, except if
+ # the user explicitly sets the number of jobs.
+ #
+ n_jobs = 1
- #
- # LaTeX PDF error code is almost useless for us:
- # any warning makes it non-zero. For kernel doc builds it
- # always return non-zero even when build succeeds.
- # So, let's do the best next thing: check if all PDF
- # files were built. If they're, print a summary and
- # return 0 at the end of this function
- #
- try:
- subprocess.run(latex_cmd + [entry.path],
- cwd=from_dir, check=True)
- except subprocess.CalledProcessError:
- pass
+ # n_jobs is either an integer or "auto". Only use it if it is a number
+ if self.n_jobs:
+ try:
+ n_jobs = int(self.n_jobs)
+ except ValueError:
+ pass
- pdf_name = name + ".pdf"
- pdf_from = os.path.join(from_dir, pdf_name)
- pdf_to = os.path.join(pdf_dir, pdf_name)
+ #
+ # When using make, jobserver.claim is the number of jobs that were
+ # used with "-j" and that aren't used by other make targets
+ #
+ with JobserverExec() as jobserver:
+ n_jobs = 1
- if os.path.exists(pdf_from):
- os.rename(pdf_from, pdf_to)
- builds[name] = os.path.relpath(pdf_to, self.builddir)
- else:
- builds[name] = "FAILED"
- build_failed = True
+ #
+ # Handle the case when a parameter is passed via command line,
+ # using it as default, if jobserver doesn't claim anything
+ #
+ if self.n_jobs:
+ try:
+ n_jobs = int(self.n_jobs)
+ except ValueError:
+ pass
- name = entry.name.removesuffix(".tex")
- max_len = max(max_len, len(name))
+ if jobserver.claim:
+ n_jobs = jobserver.claim
- if not has_tex:
- name = os.path.basename(from_dir)
- max_len = max(max_len, len(name))
- builds[name] = "FAILED (no .tex)"
- build_failed = True
+ builds, build_failed, max_len = self.pdf_parallel_build(tex_suffix,
+ latex_cmd,
+ tex_files,
+ n_jobs)
msg = "Summary"
msg += "\n" + "=" * len(msg)
--
2.51.0
