Coverage for pyquickhelper/benchhelper/benchmark.py: 94%
293 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-03 02:21 +0200
1"""
2@file
3@brief Helpers to benchmark something
4"""
5import os
6from datetime import datetime
7from time import perf_counter
8import pickle
9from ..loghelper import noLOG, CustomLog, fLOGFormat
10from ..loghelper.flog import get_relative_path
11from ..pandashelper import df2rst
12from ..texthelper import apply_template
15class BenchMark:
16 """
17 Class to help benchmarking. You should overwrite method
18 *init*, *bench*, *end*, *graphs*.
19 """
21 def __init__(self, name, clog=None, fLOG=noLOG, path_to_images=".",
22 cache_file=None, pickle_module=None, progressbar=None,
23 **params):
24 """
25 @param name name of the test
26 @param clog @see cl CustomLog or string
27 @param fLOG logging function
28 @param params extra parameters
29 @param path_to_images path to images
30 @param cache_file cache file
31 @param pickle_module pickle or dill if you need to serialize functions
32 @param progressbar relies on *tqdm*, example *tnrange*
34 If *cache_file* is specified, the class will store the results of the
35 method :meth:`bench <pyquickhelper.benchhelper.benchmark.GridBenchMark.bench>`.
36 On a second run, the function load the cache
37 and run modified or new run (in *params_list*).
38 """
39 self._fLOG = fLOG
40 self._name = name
42 if isinstance(clog, CustomLog):
43 self._clog = clog
44 elif clog is None:
45 self._clog = None
46 else:
47 self._clog = CustomLog(clog)
48 self._params = params
49 self._path_to_images = path_to_images
50 self._cache_file = cache_file
51 self._pickle = pickle_module if pickle_module is not None else pickle
52 self._progressbar = progressbar
53 self._tracelogs = []
55 ##
56 # methods to overwrite
57 ##
59 def init(self):
60 """
61 Initialisation. Overwrite this method.
62 """
63 raise NotImplementedError(
64 "It should be overwritten.") # pragma: no cover
66 def bench(self, **params):
67 """
68 Runs the benchmark. Overwrite this method.
70 @param params parameters
71 @return metrics as a dictionary, appendix as a dictionary
73 The results of this method will be cached if a *cache_file* was specified in the constructor.
74 """
75 raise NotImplementedError(
76 "It should be overwritten.") # pragma: no cover
78 def end(self):
79 """
80 Cleans. Overwrites this method.
81 """
82 raise NotImplementedError(
83 "It should be overwritten.") # pragma: no cover
85 def graphs(self, path_to_images):
86 """
87 Builds graphs after the benchmark was run.
89 @param path_to_images path to images
90 @return a list of LocalGraph
92 Every returned graph must contain a function which creates
93 the graph. The function must accepts two parameters *ax* and
94 *text*. Example:
96 ::
98 def local_graph(ax=None, text=True, figsize=(5,5)):
99 vx = ...
100 vy = ...
101 btrys = set(df["_btry"])
102 ymin = df[vy].min()
103 ymax = df[vy].max()
104 decy = (ymax - ymin) / 50
105 colors = cm.rainbow(numpy.linspace(0, 1, len(btrys)))
106 if len(btrys) == 0:
107 raise ValueError("The benchmark is empty.")
108 if ax is None:
109 fig, ax = plt.subplots(1, 1, figsize=figsize)
110 ax.grid(True)
111 for i, btry in enumerate(sorted(btrys)):
112 subset = df[df["_btry"]==btry]
113 if subset.shape[0] > 0:
114 subset.plot(x=vx, y=vy, kind="scatter", label=btry, ax=ax, color=colors[i])
115 if text:
116 tx = subset[vx].mean()
117 ty = subset[vy].mean()
118 ax.text(tx, ty + decy, btry, size='small',
119 color=colors[i], ha='center', va='bottom')
120 ax.set_xlabel(vx)
121 ax.set_ylabel(vy)
122 return ax
123 """
124 return []
126 def uncache(self, cache):
127 """
128 overwrite this method to uncache some previous run
129 """
130 pass
132 ##
133 # end of methods to overwrite
134 ##
136 class LocalGraph:
137 """
138 Information about graphs.
139 """
141 def __init__(self, func_gen, filename=None, title=None, root=None):
142 """
143 @param func_gen function generating the graph
144 @param filename filename
145 @param title title
146 @param root path should be relative to this one
147 """
148 if func_gen is None:
149 raise ValueError("func_gen cannot be None") # pragma: no cover
150 if filename is not None:
151 self.filename = filename
152 if title is not None:
153 self.title = title
154 self.root = root
155 self.func_gen = func_gen
157 def plot(self, ax=None, text=True, **kwargs):
158 """
159 Draws the graph again.
161 @param ax axis
162 @param text add text on the graph
163 @param kwargs additional parameters
164 @return axis
165 """
166 return self.func_gen(ax=ax, text=text, **kwargs)
168 def add(self, name, value):
169 """
170 Adds an attribute.
172 @param name name of the attribute
173 @param value value
174 """
175 setattr(self, name, value)
177 def to_html(self):
178 """
179 Renders as :epkg:`HTML`.
180 """
181 # deal with relatif path.
182 if hasattr(self, "filename"):
183 attr = {}
184 for k in {"title", "alt", "width", "height"}:
185 if k not in attr and hasattr(self, k):
186 attr[k if k != "title" else "alt"] = getattr(self, k)
187 merge = " ".join(f'{k}="{v}"'
188 for k, v in attr.items())
189 if self.root is not None:
190 filename = get_relative_path(
191 self.root, self.filename, exists=False, absolute=False)
192 else:
193 filename = self.filename
194 filename = filename.replace("\\", "/")
195 return f'<img src="{filename}" {merge}/>'
196 else:
197 raise NotImplementedError(
198 "only files are allowed") # pragma: no cover
200 def to_rst(self):
201 """
202 Renders as :ekg:`rst`.
203 """
204 # do not consider width or height
205 # deal with relatif path
206 if hasattr(self, "filename"):
207 if self.root is not None:
208 filename = get_relative_path(
209 self.root, self.filename, exists=False, absolute=False)
210 else:
211 filename = self.filename
212 filename = filename.replace("\\", "/")
213 return f'.. image:: {filename}'
214 else:
215 raise NotImplementedError(
216 "only files are allowed") # pragma: no cover
218 @property
219 def Name(self):
220 """
221 Returns the name of the benchmark.
222 """
223 return self._name
225 def fLOG(self, *args, **kwargs):
226 """
227 Logs something.
228 """
229 self._tracelogs.append(fLOGFormat("\n", *args, **kwargs).strip("\n"))
230 if self._clog:
231 self._clog(*args, **kwargs)
232 if self._fLOG:
233 self._fLOG(*args, **kwargs)
234 if hasattr(self, "_progressbars") and self._progressbars and len(self._progressbars) > 0:
235 br = self._progressbars[-1]
236 br.set_description(fLOGFormat( # pylint: disable=C0207
237 "\n", *args, **kwargs).strip("\n").split("\n")[0])
238 br.refresh()
240 def run(self, params_list):
241 """
242 Runs the benchmark.
244 @param params_list list of dictionaries
245 """
246 if not isinstance(params_list, list):
247 raise TypeError("params_list must be a list") # pragma: no cover
248 for di in params_list:
249 if not isinstance(di, dict):
250 raise TypeError( # pragma: no cover
251 "params_list must be a list of dictionaries")
253 # shared variables
254 cached = {}
255 meta = dict(level="BenchMark", name=self.Name, nb=len(
256 params_list), time_begin=datetime.now())
257 self._metadata = []
258 self._metadata.append(meta)
259 nb_cached = 0
261 # cache
262 def cache_():
263 "local function"
264 if self._cache_file is not None and os.path.exists(self._cache_file):
265 self.fLOG(
266 f"[BenchMark.run] retrieve cache '{self._cache_file}'")
267 with open(self._cache_file, "rb") as f:
268 cached.update(self._pickle.load(f))
269 self.fLOG("[BenchMark.run] number of cached run: {0}".format(
270 len(cached["params_list"])))
271 else:
272 if self._cache_file is not None:
273 self.fLOG(
274 f"[BenchMark.run] cache not found '{self._cache_file}'")
275 cached.update(dict(metrics=[], appendix=[], params_list=[]))
276 self.uncache(cached)
278 # run
279 def run_(pgar):
280 "local function"
281 nonlocal nb_cached
282 self._metrics = []
283 self._appendix = []
285 self.fLOG(f"[BenchMark.run] init {self.Name} do")
286 self.init()
287 self.fLOG(f"[BenchMark.run] init {self.Name} done")
288 self.fLOG(f"[BenchMark.run] start {self.Name}")
290 for i in pgbar:
291 di = params_list[i]
293 # check the cache
294 if i < len(cached["params_list"]) and cached["params_list"][i] == di:
295 can = True
296 for v in cached.values():
297 if i >= len(v):
298 # cannot cache
299 can = False
300 break
302 if can:
303 # can, it checks a file is present
304 look = "{0}.{1}.clean_cache".format(
305 self._cache_file, cached["metrics"][i]["_btry"])
306 if not os.path.exists(look):
307 can = False
308 self.fLOG(
309 f"[BenchMark.run] file '{look}' was not found --> run again.")
310 if can:
311 self._metrics.append(cached["metrics"][i])
312 self._appendix.append(cached["appendix"][i])
313 self.fLOG(
314 f"[BenchMark.run] retrieved cached {i + 1}/{len(params_list)}: {di}")
315 self.fLOG(
316 f"[BenchMark.run] file '{look}' was found.")
317 nb_cached += 1
318 continue
320 # cache is available
322 # no cache
323 self.fLOG(
324 f"[BenchMark.run] {i + 1}/{len(params_list)}: {di}")
325 dt = datetime.now()
326 cl = perf_counter()
327 tu = self.bench(**di)
328 cl = perf_counter() - cl
330 if isinstance(tu, tuple):
331 tus = [tu]
332 elif isinstance(tu, list):
333 tus = tu
334 else:
335 raise TypeError( # pragma: no cover
336 "return of method bench must be a tuple of a list")
338 # checkings
339 for tu in tus:
340 met, app = tu
341 if len(tu) != 2:
342 raise TypeError( # pragma: no cover
343 "Method run should return a tuple with 2 elements.")
344 if "_btry" not in met:
345 raise KeyError( # pragma: no cover
346 "Metrics should contain key '_btry'.")
347 if "_btry" not in app:
348 raise KeyError( # pragma: no cover
349 "Appendix should contain key '_btry'.")
351 for met, app in tus:
352 met["_date"] = dt
353 dt = datetime.now() - dt
354 if not isinstance(met, dict):
355 raise TypeError( # pragma: no cover
356 "metrics should be a dictionary")
357 if "_time" in met:
358 raise KeyError( # pragma: no cover
359 "key _time should not be the returned metrics")
360 if "_span" in met:
361 raise KeyError( # pragma: no cover
362 "key _span should not be the returned metrics")
363 if "_i" in met:
364 raise KeyError( # pragma: no cover
365 "key _i should not be in the returned metrics")
366 if "_name" in met:
367 raise KeyError( # pragma: no cover
368 "key _name should not be the returned metrics")
369 met["_time"] = cl
370 met["_span"] = dt
371 met["_i"] = i
372 met["_name"] = self.Name
373 self._metrics.append(met)
374 app["_i"] = i
375 self._appendix.append(app)
376 self.fLOG(
377 f"[BenchMark.run] {i + 1}/{len(params_list)} end {met}")
379 def graph_():
380 "local function"
381 self.fLOG(f"[BenchMark.run] graph {self.Name} do")
382 self._graphs = self.graphs(self._path_to_images)
383 if self._graphs is None or not isinstance(self._graphs, list):
384 raise TypeError( # pragma: no cover
385 "Method graphs does not return anything.")
386 for tu in self._graphs:
387 if not isinstance(tu, self.LocalGraph):
388 raise TypeError( # pragma: no cover
389 "Method graphs should return a list of LocalGraph.")
390 self.fLOG(f"[BenchMark.run] graph {self.Name} done")
391 self.fLOG(f"[BenchMark.run] Received {len(self._graphs)} graphs.")
392 self.fLOG(f"[BenchMark.run] end {self.Name} do")
393 self.end()
394 self.fLOG(f"[BenchMark.run] end {self.Name} done")
395 meta["time_end"] = datetime.now()
396 meta["nb_cached"] = nb_cached
398 # write information about run experiments
399 def final_():
400 "local function"
401 if self._cache_file is not None:
402 self.fLOG(f"[BenchMark.run] save cache '{self._cache_file}'")
403 cached = dict(metrics=self._metrics,
404 appendix=self._appendix, params_list=params_list)
405 with open(self._cache_file, "wb") as f:
406 self._pickle.dump(cached, f)
407 for di in self._metrics:
408 look = f"{self._cache_file}.{di['_btry']}.clean_cache"
409 with open(look, "w") as f:
410 f.write(
411 "Remove this file if you want to force a new run.")
412 self.fLOG(f"[BenchMark.run] wrote '{look}'.")
414 self.fLOG("[BenchMark.run] done.")
416 progress = self._progressbar if self._progressbar is not None else range
417 functions = [cache_, run_, graph_, final_]
418 pgbar0 = progress(0, len(functions))
419 if self._progressbar:
420 self._progressbars = [pgbar0]
421 for i in pgbar0:
422 if i == 1:
423 pgbar = progress(len(params_list))
424 if self._progressbar:
425 self._progressbars.append(pgbar)
426 functions[i](pgbar)
427 if self._progressbar:
428 self._progressbars.pop()
429 else:
430 functions[i]()
432 self._progressbars = None
433 return self._metrics, self._metadata
435 @property
436 def Metrics(self):
437 """
438 Returns the metrics.
439 """
440 if not hasattr(self, "_metrics"):
441 raise KeyError( # pragma: no cover
442 "Method run was not run, no metrics was found.")
443 return self._metrics
445 @property
446 def Metadata(self):
447 """
448 Returns the metrics.
449 """
450 if not hasattr(self, "_metadata"):
451 raise KeyError( # pragma: no cover
452 "Method run was not run, no metadata was found.")
453 return self._metadata
455 @property
456 def Appendix(self):
457 """
458 Returns the metrics.
459 """
460 if not hasattr(self, "_appendix"):
461 raise KeyError( # pragma: no cover
462 "Method run was not run, no metadata was found.")
463 return self._appendix
465 def to_df(self, convert=False, add_link=False, format="html"):
466 """
467 Converts the metrics into a dataframe.
469 @param convert if True, calls method *_convert* on each cell
470 @param add_link add hyperlink
471 @param format format for hyperlinks (html or rst)
472 @return dataframe
473 """
474 import pandas
475 df = pandas.DataFrame(self.Metrics)
476 if convert:
477 for c, d in zip(df.columns, df.dtypes):
478 cols = []
479 for i in range(df.shape[0]):
480 cols.append(self._convert(df, i, c, d, df.loc[i, c]))
481 df[c] = cols
482 col1 = list(sorted(_ for _ in df.columns if _.startswith("_")))
483 col2 = list(sorted(_ for _ in df.columns if not _.startswith("_")))
484 df = df[col1 + col2]
485 if add_link and "_i" in df.columns:
486 if format == "html":
487 if "_btry" in df.columns:
488 df["_btry"] = df.apply(
489 lambda row: f"<a href=\"#{row['_i']}\">{row['_btry']}</a>", axis=1)
490 df["_i"] = df["_i"].apply(
491 lambda s: '<a href="#{0}">{0}</a>'.format(s))
492 elif format == "rst":
493 if "_btry" in df.columns:
494 df["_btry"] = df.apply(
495 lambda row: f":ref:`{row['_btry']} <l-{self.Name}-{row['_i']}>`", axis=1)
496 df["_i"] = df["_i"].apply(
497 lambda s: ':ref:`{0} <l-{1}-{0}>`'.format(s, self.Name))
498 else:
499 raise ValueError( # pragma: no cover
500 "Format should be rst or html.")
501 return df
503 def meta_to_df(self, convert=False, add_link=False, format="html"):
504 """
505 Converts meta data into a dataframe
507 @param convert if True, calls method *_convert* on each cell
508 @param add_link add hyperlink
509 @param format format for hyperlinks (html or rst)
510 @return dataframe
511 """
512 import pandas
513 df = pandas.DataFrame(self.Metadata)
514 if convert:
515 for c, d in zip(df.columns, df.dtypes):
516 cols = []
517 for i in range(df.shape[0]):
518 cols.append(self._convert(df, i, c, d, df.loc[i, c]))
519 df[c] = cols
520 col1 = list(sorted(_ for _ in df.columns if _.startswith("_")))
521 col2 = list(sorted(_ for _ in df.columns if not _.startswith("_")))
522 if add_link and "_i" in df.columns:
523 if format == "html":
524 if "_btry" in df.columns:
525 df["_btry"] = df.apply(
526 lambda row: f"<a href=\"#{row['_i']}\">{row['_btry']}</a>", axis=1)
527 df["_i"] = df["_i"].apply(
528 lambda s: '<a href="#{0}">{0}</a>'.format(s))
529 elif format == "rst":
530 if "_btry" in df.columns:
531 df["_btry"] = df.apply(
532 lambda row: f":ref:`{row['_btry']} <l-{self.Name}-{row['_i']}>`", axis=1)
533 df["_i"] = df["_i"].apply(
534 lambda s: ':ref:`{0} <l-{1}-{0}>'.format(s, self.Name))
535 else:
536 raise ValueError( # pragma: no cover
537 "Format should be rst or html.")
538 return df[col1 + col2]
540 def report(self, css=None, template_html=None, template_rst=None, engine="mako", filecsv=None,
541 filehtml=None, filerst=None, params_html=None, title=None, description=None):
542 """
543 Produces a report.
545 @param css css (will take the default one if empty)
546 @param template_html template HTML (:epkg:`mako` or :epkg:`jinja2`)
547 @param template_rst template RST (:epkg:`mako` or :epkg:`jinja2`)
548 @param engine ``'mako``' or '``jinja2'``
549 @param filehtml report will written in this file if not None
550 @param filecsv metrics will be written as a flat table
551 @param filerst metrics will be written as a RST table
552 @param params_html parameter to send to function :epkg:`pandas:DataFrame.to_html`
553 @param title title (Name if any)
554 @param description add a description
555 @return dictionary {format: content}
557 You can define your own template by looking into the default ones
558 defines in this class (see the bottom of this file).
559 By default, HTML and RST report are generated.
560 """
561 if template_html is None:
562 template_html = BenchMark.default_template_html
563 if template_rst is None:
564 template_rst = BenchMark.default_template_rst
565 if css is None:
566 css = BenchMark.default_css
567 if params_html is None:
568 params_html = dict()
569 if title is None:
570 title = self.Name # pragma: no cover
571 if "escape" not in params_html:
572 params_html["escape"] = False
574 for gr in self.Graphs:
575 gr.add("root", os.path.dirname(filehtml))
577 # I don't like that too much as it is not multithreaded.
578 # Avoid truncation.
579 import pandas
581 if description is None:
582 description = "" # pragma: no cover
584 contents = {'df': self.to_df()}
586 # HTML
587 if template_html is not None and len(template_html) > 0:
588 old_width = pandas.get_option('display.max_colwidth')
589 pandas.set_option('display.max_colwidth', None)
590 res = apply_template(template_html, dict(description=description, title=title,
591 css=css, bench=self, params_html=params_html))
592 # Restore previous value.
593 pandas.set_option('display.max_colwidth', old_width)
595 if filehtml is not None:
596 with open(filehtml, "w", encoding="utf-8") as f:
597 f.write(res)
598 contents["html"] = res
600 # RST
601 if template_rst is not None and len(template_rst) > 0:
602 old_width = pandas.get_option('display.max_colwidth')
603 pandas.set_option('display.max_colwidth', None)
605 res = apply_template(template_rst, dict(description=description,
606 title=title, bench=self, df2rst=df2rst))
608 # Restore previous value.
609 pandas.set_option('display.max_colwidth', old_width)
611 with open(filerst, "w", encoding="utf-8") as f:
612 f.write(res)
613 contents["rst"] = res
615 # CSV
616 if filecsv is not None:
617 contents['df'].to_csv(
618 filecsv, encoding="utf-8", index=False, sep="\t")
619 return contents
621 def _convert(self, df, i, col, ty, value):
622 """
623 Converts a value knowing its column, its type
624 into something readable.
626 @param df dataframe
627 @param i line index
628 @param col column name
629 @param ty type
630 @param value value to convert
631 @return value
632 """
633 return value
635 @property
636 def Graphs(self):
637 """
638 Returns images of graphs.
639 """
640 if not hasattr(self, "_graphs"):
641 raise KeyError("unable to find _graphs") # pragma: no cover
642 return self._graphs
644 default_css = """
645 .datagrid table { border-collapse: collapse; border-spacing: 0; width: 100%;
646 table-layout: fixed; font-family: Verdana; font-size: 12px;
647 word-wrap: break-word; }
649 .datagrid thead {
650 cursor: pointer;
651 background: #c9dff0;
652 }
653 .datagrid thead tr th {
654 font-weight: bold;
655 padding: 12px 30px;
656 padding-left: 12px;
657 }
658 .datagrid thead tr th span {
659 padding-right: 10px;
660 background-repeat: no-repeat;
661 text-align: left;
662 }
664 .datagrid tbody tr {
665 color: #555;
666 }
667 .datagrid tbody td {
668 text-align: center;
669 padding: 10px 5px;
670 }
671 .datagrid tbody th {
672 text-align: left;
673 padding: 10px 5px;
674 }
675 """.replace(" ", "")
677 default_template_html = """
678 <html>
679 <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
680 <style>
681 ${css}
682 </style>
683 <body>
684 <h1>${title}</h1>
685 ${description}
686 <ul>
687 <li><a href="#metadata">Metadata</a></li>
688 <li><a href="#metrics">Metrics</a></li>
689 <li><a href="#graphs">Graphs</a></li>
690 <li><a href="#appendix">Appendix</a></li>
691 </ul>
692 <h2 id="metadata">Metadata</h2>
693 <div class="datagrid">
694 ${bench.meta_to_df(convert=True, add_link=True).to_html(**params_html)}
695 </div>
696 <h2 id="metrics">Metrics</h2>
697 <div class="datagrid">
698 ${bench.to_df(convert=True, add_link=True).to_html(**params_html)}
699 </div>
700 % if len(bench.Graphs) > 0:
701 <h2 id="graphs">Graphs</h2>
702 % for gr in bench.Graphs:
703 ${gr.to_html()}
704 % endfor
705 % endif
706 % if len(bench.Appendix) > 0:
707 <h2 id="appendix">Appendix</h2>
708 <div class="appendix">
709 % for met, app in zip(bench.Metrics, bench.Appendix):
710 <h3 id="${app["_i"]}">${app["_btry"]}</h3>
711 <ul>
712 % for k, v in sorted(app.items()):
713 % if isinstance(v, str) and "\\n" in v:
714 <li>I <b>${k}</b>: <pre>${v}</pre></li>
715 % else:
716 <li>I <b>${k}</b>: ${v}</li>
717 % endif
718 % endfor
719 % for k, v in sorted(met.items()):
720 % if isinstance(v, str) and "\\n" in v:
721 <li>M <b>${k}</b>: <pre>${v}</pre></li>
722 % else:
723 <li>M <b>${k}</b>: ${v}</li>
724 % endif
725 % endfor
726 </ul>
727 % endfor
728 % endif
729 </div>
730 </body>
731 </html>
732 """.replace(" ", "")
734 default_template_rst = """
736 .. _lb-${bench.Name}:
738 ${title}
739 ${"=" * len(title)}
741 .. contents::
742 :local:
744 ${description}
746 Metadata
747 --------
749 ${df2rst(bench.meta_to_df(convert=True, add_link=True, format="rst"), index=True, list_table=True)}
751 Metrics
752 --------
754 ${df2rst(bench.to_df(convert=True, add_link=True, format="rst"), index=True, list_table=True)}
756 % if len(bench.Graphs) > 0:
758 Graphs
759 ------
761 % for gr in bench.Graphs:
762 ${gr.to_rst()}
763 % endfor
765 % endif
767 % if len(bench.Appendix) > 0:
769 Appendix
770 --------
772 % for met, app in zip(bench.Metrics, bench.Appendix):
774 .. _l-${bench.Name}-${app["_i"]}:
776 ${app["_btry"]}
777 ${"+" * len(app["_btry"])}
779 % for k, v in sorted(app.items()):
780 % if isinstance(v, str) and "\\n" in v:
781 * I **${k}**:
782 ::
784 ${"\\n ".join(v.split("\\n"))}
786 % else:
787 * M **${k}**: ${v}
788 % endif
789 % endfor
791 % for k, v in sorted(met.items()):
792 % if isinstance(v, str) and "\\n" in v:
793 * **${k}**:
794 ::
796 ${"\\n ".join(v.split("\\n"))}
798 % else:
799 * **${k}**: ${v}
800 % endif
801 % endfor
803 % endfor
804 % endif
805 """.replace(" ", "")