Commit 758e68cf authored by Emmanuele Bassi's avatar Emmanuele Bassi

Merge branch 'async-generate'

Fixes: #29
parents 24b38e56 5993f113
......@@ -2,6 +2,7 @@
# SPDX-License-Identifier: Apache-2.0 OR GPL-3.0-or-later
import argparse
import concurrent.futures
import jinja2
import markdown
import os
......@@ -2067,18 +2068,32 @@ def gen_reference(config, options, repository, templates_dir, theme_config, cont
log.info(f"Generating references for: {gen_indices}")
template_symbols = {}
for section in gen_indices:
generator = all_indices.get(section, None)
if generator is None:
log.debug(f"No generator for section {section}")
continue
s = symbols.get(section, [])
if s is None:
log.debug(f"No symbols for section {section}")
continue
# Each section is isolated, so we run it into a thread pool
with concurrent.futures.ThreadPoolExecutor() as executor:
futures_to_section = {}
for section in gen_indices:
s = symbols.get(section, [])
if s is None:
log.debug(f"No symbols for section {section}")
continue
generator = all_indices.get(section, None)
if generator is None:
log.debug(f"No generator for section {section}")
continue
template_symbols[section] = generator(config, theme_config, ns_dir, jinja_env, repository, s)
f = executor.submit(generator, config, theme_config, ns_dir, jinja_env, repository, s)
futures_to_section[f] = section
for future in concurrent.futures.as_completed(futures_to_section):
section = futures_to_section[future]
try:
res = future.result()
except Exception as e:
print(f"Section {section} raised {e}")
else:
template_symbols[section] = res
ns_tmpl = jinja_env.get_template(theme_config.namespace_template)
ns_file = os.path.join(ns_dir, "index.html")
......@@ -2092,29 +2107,34 @@ def gen_reference(config, options, repository, templates_dir, theme_config, cont
"content_files": content_files,
}))
if config.devhelp:
devhelp_file = os.path.join(ns_dir, f"{namespace.name}-{namespace.version}.devhelp2")
log.info(f"Creating DevHelp file for {namespace.name}.{namespace.version}: {devhelp_file}")
res = gen_devhelp(config, repository, namespace, template_symbols, content_files)
res.write(devhelp_file, encoding="UTF-8")
copy_files = []
if theme_config.css is not None:
log.info(f"Copying style from {theme_dir} to {ns_dir}")
style_src = os.path.join(theme_dir, theme_config.css)
style_dst = os.path.join(ns_dir, theme_config.css)
shutil.copyfile(style_src, style_dst)
copy_files.append((style_src, style_dst))
for extra_file in theme_config.extra_files:
log.info(f"Copying extra file {extra_file} from {theme_dir} to {ns_dir}")
src = os.path.join(theme_dir, extra_file)
dst = os.path.join(ns_dir, extra_file)
shutil.copyfile(src, dst)
copy_files.append((src, dst))
for (src, dst) in content_images:
log.info(f"Copying content image {src}: {dst}")
copy_files.extend(content_images)
def copy_worker(src, dst):
log.info(f"Copying file {src}: {dst}")
dst_dir = os.path.dirname(dst)
os.makedirs(dst_dir, exist_ok=True)
shutil.copyfile(src, dst)
shutil.copy(src, dst)
if config.devhelp:
devhelp_file = os.path.join(ns_dir, f"{namespace.name}-{namespace.version}.devhelp2")
log.info(f"Creating DevHelp file for {namespace.name}.{namespace.version}: {devhelp_file}")
res = gen_devhelp(config, repository, namespace, template_symbols, content_files)
res.write(devhelp_file, encoding="UTF-8")
with concurrent.futures.ThreadPoolExecutor() as executor:
for (src, dst) in copy_files:
executor.submit(copy_worker, src, dst)
def add_args(parser):
......
......@@ -361,7 +361,7 @@ def preprocess_docs(text, namespace, summary=False, md=None, extensions=[]):
extensions=md_ext,
extension_configs=MD_EXTENSIONS_CONF)
else:
text = md.convert("\n".join(processed_text))
text = md.reset().convert("\n".join(processed_text))
return Markup(typogrify(text, ignore_tags=['h1', 'h2', 'h3', 'h4']))
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment