1 /**
2 * Ddoc documentation generation.
3 *
4 * Specification: $(LINK2 https://dlang.org/spec/ddoc.html, Documentation Generator)
5 *
6 * Copyright: Copyright (C) 1999-2022 by The D Language Foundation, All Rights Reserved
7 * Authors: $(LINK2 https://www.digitalmars.com, Walter Bright)
8 * License: $(LINK2 https://www.boost.org/LICENSE_1_0.txt, Boost License 1.0)
9 * Source: $(LINK2 https://github.com/dlang/dmd/blob/master/src/dmd/doc.d, _doc.d)
10 * Documentation: https://dlang.org/phobos/dmd_doc.html
11 * Coverage: https://codecov.io/gh/dlang/dmd/src/master/src/dmd/doc.d
12 */
13
14 module dmd.doc;
15
16 import core.stdc.ctype;
17 import core.stdc.stdlib;
18 import core.stdc.stdio;
19 import core.stdc.string;
20 import core.stdc.time;
21 import dmd.aggregate;
22 import dmd.arraytypes;
23 import dmd.astenums;
24 import dmd.attrib;
25 import dmd.cond;
26 import dmd.dclass;
27 import dmd.declaration;
28 import dmd.denum;
29 import dmd.dimport;
30 import dmd.dmacro;
31 import dmd.dmodule;
32 import dmd.dscope;
33 import dmd.dstruct;
34 import dmd.dsymbol;
35 import dmd.dsymbolsem;
36 import dmd.dtemplate;
37 import dmd.errors;
38 import dmd.func;
39 import dmd.globals;
40 import dmd.hdrgen;
41 import dmd.id;
42 import dmd.identifier;
43 import dmd.lexer;
44 import dmd.mtype;
45 import dmd.root.array;
46 import dmd.root.file;
47 import dmd.root.filename;
48 import dmd.common.outbuffer;
49 import dmd.root.port;
50 import dmd.root.rmem;
51 import dmd.root.string;
52 import dmd.root.utf;
53 import dmd.tokens;
54 import dmd.utils;
55 import dmd.visitor;
56
57 struct Escape
58 {
59 const(char)[][char.max] strings;
60
61 /***************************************
62 * Find character string to replace c with.
63 */
escapeCharEscape64 const(char)[] escapeChar(char c)
65 {
66 version (all)
67 {
68 //printf("escapeChar('%c') => %p, %p\n", c, strings, strings[c].ptr);
69 return strings[c];
70 }
71 else
72 {
73 const(char)[] s;
74 switch (c)
75 {
76 case '<':
77 s = "<";
78 break;
79 case '>':
80 s = ">";
81 break;
82 case '&':
83 s = "&";
84 break;
85 default:
86 s = null;
87 break;
88 }
89 return s;
90 }
91 }
92 }
93
94 /***********************************************************
95 */
96 private class Section
97 {
98 const(char)[] name;
99 const(char)[] body_;
100 int nooutput;
101
toString()102 override string toString() const
103 {
104 assert(0);
105 }
106
write(Loc loc,DocComment * dc,Scope * sc,Dsymbols * a,OutBuffer * buf)107 void write(Loc loc, DocComment* dc, Scope* sc, Dsymbols* a, OutBuffer* buf)
108 {
109 assert(a.dim);
110 if (name.length)
111 {
112 static immutable table =
113 [
114 "AUTHORS",
115 "BUGS",
116 "COPYRIGHT",
117 "DATE",
118 "DEPRECATED",
119 "EXAMPLES",
120 "HISTORY",
121 "LICENSE",
122 "RETURNS",
123 "SEE_ALSO",
124 "STANDARDS",
125 "THROWS",
126 "VERSION",
127 ];
128 foreach (entry; table)
129 {
130 if (iequals(entry, name))
131 {
132 buf.printf("$(DDOC_%s ", entry.ptr);
133 goto L1;
134 }
135 }
136 buf.writestring("$(DDOC_SECTION ");
137 // Replace _ characters with spaces
138 buf.writestring("$(DDOC_SECTION_H ");
139 size_t o = buf.length;
140 foreach (char c; name)
141 buf.writeByte((c == '_') ? ' ' : c);
142 escapeStrayParenthesis(loc, buf, o, false);
143 buf.writestring(")");
144 }
145 else
146 {
147 buf.writestring("$(DDOC_DESCRIPTION ");
148 }
149 L1:
150 size_t o = buf.length;
151 buf.write(body_);
152 escapeStrayParenthesis(loc, buf, o, true);
153 highlightText(sc, a, loc, *buf, o);
154 buf.writestring(")");
155 }
156 }
157
158 /***********************************************************
159 */
160 private final class ParamSection : Section
161 {
write(Loc loc,DocComment * dc,Scope * sc,Dsymbols * a,OutBuffer * buf)162 override void write(Loc loc, DocComment* dc, Scope* sc, Dsymbols* a, OutBuffer* buf)
163 {
164 assert(a.dim);
165 Dsymbol s = (*a)[0]; // test
166 const(char)* p = body_.ptr;
167 size_t len = body_.length;
168 const(char)* pend = p + len;
169 const(char)* tempstart = null;
170 size_t templen = 0;
171 const(char)* namestart = null;
172 size_t namelen = 0; // !=0 if line continuation
173 const(char)* textstart = null;
174 size_t textlen = 0;
175 size_t paramcount = 0;
176 buf.writestring("$(DDOC_PARAMS ");
177 while (p < pend)
178 {
179 // Skip to start of macro
180 while (1)
181 {
182 switch (*p)
183 {
184 case ' ':
185 case '\t':
186 p++;
187 continue;
188 case '\n':
189 p++;
190 goto Lcont;
191 default:
192 if (isIdStart(p) || isCVariadicArg(p[0 .. cast(size_t)(pend - p)]))
193 break;
194 if (namelen)
195 goto Ltext;
196 // continuation of prev macro
197 goto Lskipline;
198 }
199 break;
200 }
201 tempstart = p;
202 while (isIdTail(p))
203 p += utfStride(p);
204 if (isCVariadicArg(p[0 .. cast(size_t)(pend - p)]))
205 p += 3;
206 templen = p - tempstart;
207 while (*p == ' ' || *p == '\t')
208 p++;
209 if (*p != '=')
210 {
211 if (namelen)
212 goto Ltext;
213 // continuation of prev macro
214 goto Lskipline;
215 }
216 p++;
217 if (namelen)
218 {
219 // Output existing param
220 L1:
221 //printf("param '%.*s' = '%.*s'\n", cast(int)namelen, namestart, cast(int)textlen, textstart);
222 ++paramcount;
223 HdrGenState hgs;
224 buf.writestring("$(DDOC_PARAM_ROW ");
225 {
226 buf.writestring("$(DDOC_PARAM_ID ");
227 {
228 size_t o = buf.length;
229 Parameter fparam = isFunctionParameter(a, namestart, namelen);
230 if (!fparam)
231 {
232 // Comments on a template might refer to function parameters within.
233 // Search the parameters of nested eponymous functions (with the same name.)
234 fparam = isEponymousFunctionParameter(a, namestart, namelen);
235 }
236 bool isCVariadic = isCVariadicParameter(a, namestart[0 .. namelen]);
237 if (isCVariadic)
238 {
239 buf.writestring("...");
240 }
241 else if (fparam && fparam.type && fparam.ident)
242 {
243 .toCBuffer(fparam.type, buf, fparam.ident, &hgs);
244 }
245 else
246 {
247 if (isTemplateParameter(a, namestart, namelen))
248 {
249 // 10236: Don't count template parameters for params check
250 --paramcount;
251 }
252 else if (!fparam)
253 {
254 warning(s.loc, "Ddoc: function declaration has no parameter '%.*s'", cast(int)namelen, namestart);
255 }
256 buf.write(namestart[0 .. namelen]);
257 }
258 escapeStrayParenthesis(loc, buf, o, true);
259 highlightCode(sc, a, *buf, o);
260 }
261 buf.writestring(")");
262 buf.writestring("$(DDOC_PARAM_DESC ");
263 {
264 size_t o = buf.length;
265 buf.write(textstart[0 .. textlen]);
266 escapeStrayParenthesis(loc, buf, o, true);
267 highlightText(sc, a, loc, *buf, o);
268 }
269 buf.writestring(")");
270 }
271 buf.writestring(")");
272 namelen = 0;
273 if (p >= pend)
274 break;
275 }
276 namestart = tempstart;
277 namelen = templen;
278 while (*p == ' ' || *p == '\t')
279 p++;
280 textstart = p;
281 Ltext:
282 while (*p != '\n')
283 p++;
284 textlen = p - textstart;
285 p++;
286 Lcont:
287 continue;
288 Lskipline:
289 // Ignore this line
290 while (*p++ != '\n')
291 {
292 }
293 }
294 if (namelen)
295 goto L1;
296 // write out last one
297 buf.writestring(")");
298 TypeFunction tf = a.dim == 1 ? isTypeFunction(s) : null;
299 if (tf)
300 {
301 size_t pcount = (tf.parameterList.parameters ? tf.parameterList.parameters.dim : 0) +
302 cast(int)(tf.parameterList.varargs == VarArg.variadic);
303 if (pcount != paramcount)
304 {
305 warning(s.loc, "Ddoc: parameter count mismatch, expected %llu, got %llu",
306 cast(ulong) pcount, cast(ulong) paramcount);
307 if (paramcount == 0)
308 {
309 // Chances are someone messed up the format
310 warningSupplemental(s.loc, "Note that the format is `param = description`");
311 }
312 }
313 }
314 }
315 }
316
317 /***********************************************************
318 */
319 private final class MacroSection : Section
320 {
write(Loc loc,DocComment * dc,Scope * sc,Dsymbols * a,OutBuffer * buf)321 override void write(Loc loc, DocComment* dc, Scope* sc, Dsymbols* a, OutBuffer* buf)
322 {
323 //printf("MacroSection::write()\n");
324 DocComment.parseMacros(dc.escapetable, *dc.pmacrotable, body_);
325 }
326 }
327
328 private alias Sections = Array!(Section);
329
330 // Workaround for missing Parameter instance for variadic params. (it's unnecessary to instantiate one).
isCVariadicParameter(Dsymbols * a,const (char)[]p)331 private bool isCVariadicParameter(Dsymbols* a, const(char)[] p)
332 {
333 foreach (member; *a)
334 {
335 TypeFunction tf = isTypeFunction(member);
336 if (tf && tf.parameterList.varargs == VarArg.variadic && p == "...")
337 return true;
338 }
339 return false;
340 }
341
getEponymousMember(TemplateDeclaration td)342 private Dsymbol getEponymousMember(TemplateDeclaration td)
343 {
344 if (!td.onemember)
345 return null;
346 if (AggregateDeclaration ad = td.onemember.isAggregateDeclaration())
347 return ad;
348 if (FuncDeclaration fd = td.onemember.isFuncDeclaration())
349 return fd;
350 if (auto em = td.onemember.isEnumMember())
351 return null; // Keep backward compatibility. See compilable/ddoc9.d
352 if (VarDeclaration vd = td.onemember.isVarDeclaration())
353 return td.constraint ? null : vd;
354 return null;
355 }
356
getEponymousParent(Dsymbol s)357 private TemplateDeclaration getEponymousParent(Dsymbol s)
358 {
359 if (!s.parent)
360 return null;
361 TemplateDeclaration td = s.parent.isTemplateDeclaration();
362 return (td && getEponymousMember(td)) ? td : null;
363 }
364
365 private immutable ddoc_default = import("default_ddoc_theme." ~ ddoc_ext);
366 private immutable ddoc_decl_s = "$(DDOC_DECL ";
367 private immutable ddoc_decl_e = ")\n";
368 private immutable ddoc_decl_dd_s = "$(DDOC_DECL_DD ";
369 private immutable ddoc_decl_dd_e = ")\n";
370
371 /****************************************************
372 */
gendocfile(Module m)373 extern(C++) void gendocfile(Module m)
374 {
375 __gshared OutBuffer mbuf;
376 __gshared int mbuf_done;
377 OutBuffer buf;
378 //printf("Module::gendocfile()\n");
379 if (!mbuf_done) // if not already read the ddoc files
380 {
381 mbuf_done = 1;
382 // Use our internal default
383 mbuf.writestring(ddoc_default);
384 // Override with DDOCFILE specified in the sc.ini file
385 char* p = getenv("DDOCFILE");
386 if (p)
387 global.params.ddocfiles.shift(p);
388 // Override with the ddoc macro files from the command line
389 for (size_t i = 0; i < global.params.ddocfiles.dim; i++)
390 {
391 auto buffer = readFile(m.loc, global.params.ddocfiles[i]);
392 // BUG: convert file contents to UTF-8 before use
393 const data = buffer.data;
394 //printf("file: '%.*s'\n", cast(int)data.length, data.ptr);
395 mbuf.write(data);
396 }
397 }
398 DocComment.parseMacros(m.escapetable, m.macrotable, mbuf[]);
399 Scope* sc = Scope.createGlobal(m); // create root scope
400 DocComment* dc = DocComment.parse(m, m.comment);
401 dc.pmacrotable = &m.macrotable;
402 dc.escapetable = m.escapetable;
403 sc.lastdc = dc;
404 // Generate predefined macros
405 // Set the title to be the name of the module
406 {
407 const p = m.toPrettyChars().toDString;
408 m.macrotable.define("TITLE", p);
409 }
410 // Set time macros
411 {
412 time_t t;
413 time(&t);
414 char* p = ctime(&t);
415 p = mem.xstrdup(p);
416 m.macrotable.define("DATETIME", p.toDString());
417 m.macrotable.define("YEAR", p[20 .. 20 + 4]);
418 }
419 const srcfilename = m.srcfile.toString();
420 m.macrotable.define("SRCFILENAME", srcfilename);
421 const docfilename = m.docfile.toString();
422 m.macrotable.define("DOCFILENAME", docfilename);
423 if (dc.copyright)
424 {
425 dc.copyright.nooutput = 1;
426 m.macrotable.define("COPYRIGHT", dc.copyright.body_);
427 }
428 if (m.filetype == FileType.ddoc)
429 {
430 const ploc = m.md ? &m.md.loc : &m.loc;
431 const loc = Loc(ploc.filename ? ploc.filename : srcfilename.ptr,
432 ploc.linnum,
433 ploc.charnum);
434
435 size_t commentlen = strlen(cast(char*)m.comment);
436 Dsymbols a;
437 // https://issues.dlang.org/show_bug.cgi?id=9764
438 // Don't push m in a, to prevent emphasize ddoc file name.
439 if (dc.macros)
440 {
441 commentlen = dc.macros.name.ptr - m.comment;
442 dc.macros.write(loc, dc, sc, &a, &buf);
443 }
444 buf.write(m.comment[0 .. commentlen]);
445 highlightText(sc, &a, loc, buf, 0);
446 }
447 else
448 {
449 Dsymbols a;
450 a.push(m);
451 dc.writeSections(sc, &a, &buf);
452 emitMemberComments(m, buf, sc);
453 }
454 //printf("BODY= '%.*s'\n", cast(int)buf.length, buf.data);
455 m.macrotable.define("BODY", buf[]);
456 OutBuffer buf2;
457 buf2.writestring("$(DDOC)");
458 size_t end = buf2.length;
459 m.macrotable.expand(buf2, 0, end, null);
460 version (all)
461 {
462 /* Remove all the escape sequences from buf2,
463 * and make CR-LF the newline.
464 */
465 {
466 const slice = buf2[];
467 buf.setsize(0);
468 buf.reserve(slice.length);
469 auto p = slice.ptr;
470 for (size_t j = 0; j < slice.length; j++)
471 {
472 char c = p[j];
473 if (c == 0xFF && j + 1 < slice.length)
474 {
475 j++;
476 continue;
477 }
478 if (c == '\n')
479 buf.writeByte('\r');
480 else if (c == '\r')
481 {
482 buf.writestring("\r\n");
483 if (j + 1 < slice.length && p[j + 1] == '\n')
484 {
485 j++;
486 }
487 continue;
488 }
489 buf.writeByte(c);
490 }
491 }
492 writeFile(m.loc, m.docfile.toString(), buf[]);
493 }
494 else
495 {
496 /* Remove all the escape sequences from buf2
497 */
498 {
499 size_t i = 0;
500 char* p = buf2.data;
501 for (size_t j = 0; j < buf2.length; j++)
502 {
503 if (p[j] == 0xFF && j + 1 < buf2.length)
504 {
505 j++;
506 continue;
507 }
508 p[i] = p[j];
509 i++;
510 }
511 buf2.setsize(i);
512 }
513 writeFile(m.loc, m.docfile.toString(), buf2[]);
514 }
515 }
516
517 /****************************************************
518 * Having unmatched parentheses can hose the output of Ddoc,
519 * as the macros depend on properly nested parentheses.
520 * This function replaces all ( with $(LPAREN) and ) with $(RPAREN)
521 * to preserve text literally. This also means macros in the
522 * text won't be expanded.
523 */
escapeDdocString(OutBuffer * buf,size_t start)524 void escapeDdocString(OutBuffer* buf, size_t start)
525 {
526 for (size_t u = start; u < buf.length; u++)
527 {
528 char c = (*buf)[u];
529 switch (c)
530 {
531 case '$':
532 buf.remove(u, 1);
533 buf.insert(u, "$(DOLLAR)");
534 u += 8;
535 break;
536 case '(':
537 buf.remove(u, 1); //remove the (
538 buf.insert(u, "$(LPAREN)"); //insert this instead
539 u += 8; //skip over newly inserted macro
540 break;
541 case ')':
542 buf.remove(u, 1); //remove the )
543 buf.insert(u, "$(RPAREN)"); //insert this instead
544 u += 8; //skip over newly inserted macro
545 break;
546 default:
547 break;
548 }
549 }
550 }
551
552 /****************************************************
553 * Having unmatched parentheses can hose the output of Ddoc,
554 * as the macros depend on properly nested parentheses.
555 *
556 * Fix by replacing unmatched ( with $(LPAREN) and unmatched ) with $(RPAREN).
557 *
558 * Params:
559 * loc = source location of start of text. It is a mutable copy to allow incrementing its linenum, for printing the correct line number when an error is encountered in a multiline block of ddoc.
560 * buf = an OutBuffer containing the DDoc
561 * start = the index within buf to start replacing unmatched parentheses
562 * respectBackslashEscapes = if true, always replace parentheses that are
563 * directly preceeded by a backslash with $(LPAREN) or $(RPAREN) instead of
564 * counting them as stray parentheses
565 */
escapeStrayParenthesis(Loc loc,OutBuffer * buf,size_t start,bool respectBackslashEscapes)566 private void escapeStrayParenthesis(Loc loc, OutBuffer* buf, size_t start, bool respectBackslashEscapes)
567 {
568 uint par_open = 0;
569 char inCode = 0;
570 bool atLineStart = true;
571 for (size_t u = start; u < buf.length; u++)
572 {
573 char c = (*buf)[u];
574 switch (c)
575 {
576 case '(':
577 if (!inCode)
578 par_open++;
579 atLineStart = false;
580 break;
581 case ')':
582 if (!inCode)
583 {
584 if (par_open == 0)
585 {
586 //stray ')'
587 warning(loc, "Ddoc: Stray ')'. This may cause incorrect Ddoc output. Use $(RPAREN) instead for unpaired right parentheses.");
588 buf.remove(u, 1); //remove the )
589 buf.insert(u, "$(RPAREN)"); //insert this instead
590 u += 8; //skip over newly inserted macro
591 }
592 else
593 par_open--;
594 }
595 atLineStart = false;
596 break;
597 case '\n':
598 atLineStart = true;
599 version (none)
600 {
601 // For this to work, loc must be set to the beginning of the passed
602 // text which is currently not possible
603 // (loc is set to the Loc of the Dsymbol)
604 loc.linnum++;
605 }
606 break;
607 case ' ':
608 case '\r':
609 case '\t':
610 break;
611 case '-':
612 case '`':
613 case '~':
614 // Issue 15465: don't try to escape unbalanced parens inside code
615 // blocks.
616 int numdash = 1;
617 for (++u; u < buf.length && (*buf)[u] == c; ++u)
618 ++numdash;
619 --u;
620 if (c == '`' || (atLineStart && numdash >= 3))
621 {
622 if (inCode == c)
623 inCode = 0;
624 else if (!inCode)
625 inCode = c;
626 }
627 atLineStart = false;
628 break;
629 case '\\':
630 // replace backslash-escaped parens with their macros
631 if (!inCode && respectBackslashEscapes && u+1 < buf.length && global.params.markdown)
632 {
633 if ((*buf)[u+1] == '(' || (*buf)[u+1] == ')')
634 {
635 const paren = (*buf)[u+1] == '(' ? "$(LPAREN)" : "$(RPAREN)";
636 buf.remove(u, 2); //remove the \)
637 buf.insert(u, paren); //insert this instead
638 u += 8; //skip over newly inserted macro
639 }
640 else if ((*buf)[u+1] == '\\')
641 ++u;
642 }
643 break;
644 default:
645 atLineStart = false;
646 break;
647 }
648 }
649 if (par_open) // if any unmatched lparens
650 {
651 par_open = 0;
652 for (size_t u = buf.length; u > start;)
653 {
654 u--;
655 char c = (*buf)[u];
656 switch (c)
657 {
658 case ')':
659 par_open++;
660 break;
661 case '(':
662 if (par_open == 0)
663 {
664 //stray '('
665 warning(loc, "Ddoc: Stray '('. This may cause incorrect Ddoc output. Use $(LPAREN) instead for unpaired left parentheses.");
666 buf.remove(u, 1); //remove the (
667 buf.insert(u, "$(LPAREN)"); //insert this instead
668 }
669 else
670 par_open--;
671 break;
672 default:
673 break;
674 }
675 }
676 }
677 }
678
679 // Basically, this is to skip over things like private{} blocks in a struct or
680 // class definition that don't add any components to the qualified name.
skipNonQualScopes(Scope * sc)681 private Scope* skipNonQualScopes(Scope* sc)
682 {
683 while (sc && !sc.scopesym)
684 sc = sc.enclosing;
685 return sc;
686 }
687
emitAnchorName(ref OutBuffer buf,Dsymbol s,Scope * sc,bool includeParent)688 private bool emitAnchorName(ref OutBuffer buf, Dsymbol s, Scope* sc, bool includeParent)
689 {
690 if (!s || s.isPackage() || s.isModule())
691 return false;
692 // Add parent names first
693 bool dot = false;
694 auto eponymousParent = getEponymousParent(s);
695 if (includeParent && s.parent || eponymousParent)
696 dot = emitAnchorName(buf, s.parent, sc, includeParent);
697 else if (includeParent && sc)
698 dot = emitAnchorName(buf, sc.scopesym, skipNonQualScopes(sc.enclosing), includeParent);
699 // Eponymous template members can share the parent anchor name
700 if (eponymousParent)
701 return dot;
702 if (dot)
703 buf.writeByte('.');
704 // Use "this" not "__ctor"
705 TemplateDeclaration td;
706 if (s.isCtorDeclaration() || ((td = s.isTemplateDeclaration()) !is null && td.onemember && td.onemember.isCtorDeclaration()))
707 {
708 buf.writestring("this");
709 }
710 else
711 {
712 /* We just want the identifier, not overloads like TemplateDeclaration::toChars.
713 * We don't want the template parameter list and constraints. */
714 buf.writestring(s.Dsymbol.toChars());
715 }
716 return true;
717 }
718
719 private void emitAnchor(ref OutBuffer buf, Dsymbol s, Scope* sc, bool forHeader = false)
720 {
721 Identifier ident;
722 {
723 OutBuffer anc;
724 emitAnchorName(anc, s, skipNonQualScopes(sc), true);
725 ident = Identifier.idPool(anc[]);
726 }
727
728 auto pcount = cast(void*)ident in sc.anchorCounts;
729 typeof(*pcount) count;
730 if (!forHeader)
731 {
732 if (pcount)
733 {
734 // Existing anchor,
735 // don't write an anchor for matching consecutive ditto symbols
736 TemplateDeclaration td = getEponymousParent(s);
737 if (sc.prevAnchor == ident && sc.lastdc && (isDitto(s.comment) || (td && isDitto(td.comment))))
738 return;
739
740 count = ++*pcount;
741 }
742 else
743 {
744 sc.anchorCounts[cast(void*)ident] = 1;
745 count = 1;
746 }
747 }
748
749 // cache anchor name
750 sc.prevAnchor = ident;
751 auto macroName = forHeader ? "DDOC_HEADER_ANCHOR" : "DDOC_ANCHOR";
752
753 if (auto imp = s.isImport())
754 {
755 // For example: `public import core.stdc.string : memcpy, memcmp;`
756 if (imp.aliases.dim > 0)
757 {
758 for(int i = 0; i < imp.aliases.dim; i++)
759 {
760 // Need to distinguish between
761 // `public import core.stdc.string : memcpy, memcmp;` and
762 // `public import core.stdc.string : copy = memcpy, compare = memcmp;`
763 auto a = imp.aliases[i];
764 auto id = a ? a : imp.names[i];
765 auto loc = Loc.init;
766 if (auto symFromId = sc.search(loc, id, null))
767 {
768 emitAnchor(buf, symFromId, sc, forHeader);
769 }
770 }
771 }
772 else
773 {
774 // For example: `public import str = core.stdc.string;`
775 if (imp.aliasId)
776 {
777 auto symbolName = imp.aliasId.toString();
778
779 buf.printf("$(%.*s %.*s", cast(int) macroName.length, macroName.ptr,
780 cast(int) symbolName.length, symbolName.ptr);
781
782 if (forHeader)
783 {
784 buf.printf(", %.*s", cast(int) symbolName.length, symbolName.ptr);
785 }
786 }
787 else
788 {
789 // The general case: `public import core.stdc.string;`
790
791 // fully qualify imports so `core.stdc.string` doesn't appear as `core`
printFullyQualifiedImport()792 void printFullyQualifiedImport()
793 {
794 foreach (const pid; imp.packages)
795 {
796 buf.printf("%s.", pid.toChars());
797 }
798 buf.writestring(imp.id.toString());
799 }
800
801 buf.printf("$(%.*s ", cast(int) macroName.length, macroName.ptr);
802 printFullyQualifiedImport();
803
804 if (forHeader)
805 {
806 buf.printf(", ");
807 printFullyQualifiedImport();
808 }
809 }
810
811 buf.writeByte(')');
812 }
813 }
814 else
815 {
816 auto symbolName = ident.toString();
817 buf.printf("$(%.*s %.*s", cast(int) macroName.length, macroName.ptr,
818 cast(int) symbolName.length, symbolName.ptr);
819
820 // only append count once there's a duplicate
821 if (count > 1)
822 buf.printf(".%u", count);
823
824 if (forHeader)
825 {
826 Identifier shortIdent;
827 {
828 OutBuffer anc;
829 emitAnchorName(anc, s, skipNonQualScopes(sc), false);
830 shortIdent = Identifier.idPool(anc[]);
831 }
832
833 auto shortName = shortIdent.toString();
834 buf.printf(", %.*s", cast(int) shortName.length, shortName.ptr);
835 }
836
837 buf.writeByte(')');
838 }
839 }
840
841 /******************************* emitComment **********************************/
842
843 /** Get leading indentation from 'src' which represents lines of code. */
getCodeIndent(const (char)* src)844 private size_t getCodeIndent(const(char)* src)
845 {
846 while (src && (*src == '\r' || *src == '\n'))
847 ++src; // skip until we find the first non-empty line
848 size_t codeIndent = 0;
849 while (src && (*src == ' ' || *src == '\t'))
850 {
851 codeIndent++;
852 src++;
853 }
854 return codeIndent;
855 }
856
857 /** Recursively expand template mixin member docs into the scope. */
expandTemplateMixinComments(TemplateMixin tm,ref OutBuffer buf,Scope * sc)858 private void expandTemplateMixinComments(TemplateMixin tm, ref OutBuffer buf, Scope* sc)
859 {
860 if (!tm.semanticRun)
861 tm.dsymbolSemantic(sc);
862 TemplateDeclaration td = (tm && tm.tempdecl) ? tm.tempdecl.isTemplateDeclaration() : null;
863 if (td && td.members)
864 {
865 for (size_t i = 0; i < td.members.dim; i++)
866 {
867 Dsymbol sm = (*td.members)[i];
868 TemplateMixin tmc = sm.isTemplateMixin();
869 if (tmc && tmc.comment)
870 expandTemplateMixinComments(tmc, buf, sc);
871 else
872 emitComment(sm, buf, sc);
873 }
874 }
875 }
876
emitMemberComments(ScopeDsymbol sds,ref OutBuffer buf,Scope * sc)877 private void emitMemberComments(ScopeDsymbol sds, ref OutBuffer buf, Scope* sc)
878 {
879 if (!sds.members)
880 return;
881 //printf("ScopeDsymbol::emitMemberComments() %s\n", toChars());
882 const(char)[] m = "$(DDOC_MEMBERS ";
883 if (sds.isTemplateDeclaration())
884 m = "$(DDOC_TEMPLATE_MEMBERS ";
885 else if (sds.isClassDeclaration())
886 m = "$(DDOC_CLASS_MEMBERS ";
887 else if (sds.isStructDeclaration())
888 m = "$(DDOC_STRUCT_MEMBERS ";
889 else if (sds.isEnumDeclaration())
890 m = "$(DDOC_ENUM_MEMBERS ";
891 else if (sds.isModule())
892 m = "$(DDOC_MODULE_MEMBERS ";
893 size_t offset1 = buf.length; // save starting offset
894 buf.writestring(m);
895 size_t offset2 = buf.length; // to see if we write anything
896 sc = sc.push(sds);
897 for (size_t i = 0; i < sds.members.dim; i++)
898 {
899 Dsymbol s = (*sds.members)[i];
900 //printf("\ts = '%s'\n", s.toChars());
901 // only expand if parent is a non-template (semantic won't work)
902 if (s.comment && s.isTemplateMixin() && s.parent && !s.parent.isTemplateDeclaration())
903 expandTemplateMixinComments(cast(TemplateMixin)s, buf, sc);
904 emitComment(s, buf, sc);
905 }
906 emitComment(null, buf, sc);
907 sc.pop();
908 if (buf.length == offset2)
909 {
910 /* Didn't write out any members, so back out last write
911 */
912 buf.setsize(offset1);
913 }
914 else
915 buf.writestring(")");
916 }
917
emitVisibility(ref OutBuffer buf,Import i)918 private void emitVisibility(ref OutBuffer buf, Import i)
919 {
920 // imports are private by default, which is different from other declarations
921 // so they should explicitly show their visibility
922 emitVisibility(buf, i.visibility);
923 }
924
emitVisibility(ref OutBuffer buf,Declaration d)925 private void emitVisibility(ref OutBuffer buf, Declaration d)
926 {
927 auto vis = d.visibility;
928 if (vis.kind != Visibility.Kind.undefined && vis.kind != Visibility.Kind.public_)
929 {
930 emitVisibility(buf, vis);
931 }
932 }
933
emitVisibility(ref OutBuffer buf,Visibility vis)934 private void emitVisibility(ref OutBuffer buf, Visibility vis)
935 {
936 visibilityToBuffer(&buf, vis);
937 buf.writeByte(' ');
938 }
939
emitComment(Dsymbol s,ref OutBuffer buf,Scope * sc)940 private void emitComment(Dsymbol s, ref OutBuffer buf, Scope* sc)
941 {
942 extern (C++) final class EmitComment : Visitor
943 {
944 alias visit = Visitor.visit;
945 public:
946 OutBuffer* buf;
947 Scope* sc;
948
949 extern (D) this(ref OutBuffer buf, Scope* sc)
950 {
951 this.buf = &buf;
952 this.sc = sc;
953 }
954
955 override void visit(Dsymbol)
956 {
957 }
958
959 override void visit(InvariantDeclaration)
960 {
961 }
962
963 override void visit(UnitTestDeclaration)
964 {
965 }
966
967 override void visit(PostBlitDeclaration)
968 {
969 }
970
971 override void visit(DtorDeclaration)
972 {
973 }
974
975 override void visit(StaticCtorDeclaration)
976 {
977 }
978
979 override void visit(StaticDtorDeclaration)
980 {
981 }
982
983 override void visit(TypeInfoDeclaration)
984 {
985 }
986
987 void emit(Scope* sc, Dsymbol s, const(char)* com)
988 {
989 if (s && sc.lastdc && isDitto(com))
990 {
991 sc.lastdc.a.push(s);
992 return;
993 }
994 // Put previous doc comment if exists
995 if (DocComment* dc = sc.lastdc)
996 {
997 assert(dc.a.dim > 0, "Expects at least one declaration for a" ~
998 "documentation comment");
999
1000 auto symbol = dc.a[0];
1001
1002 buf.writestring("$(DDOC_MEMBER");
1003 buf.writestring("$(DDOC_MEMBER_HEADER");
1004 emitAnchor(*buf, symbol, sc, true);
1005 buf.writeByte(')');
1006
1007 // Put the declaration signatures as the document 'title'
1008 buf.writestring(ddoc_decl_s);
1009 for (size_t i = 0; i < dc.a.dim; i++)
1010 {
1011 Dsymbol sx = dc.a[i];
1012 // the added linebreaks in here make looking at multiple
1013 // signatures more appealing
1014 if (i == 0)
1015 {
1016 size_t o = buf.length;
1017 toDocBuffer(sx, *buf, sc);
1018 highlightCode(sc, sx, *buf, o);
1019 buf.writestring("$(DDOC_OVERLOAD_SEPARATOR)");
1020 continue;
1021 }
1022 buf.writestring("$(DDOC_DITTO ");
1023 {
1024 size_t o = buf.length;
1025 toDocBuffer(sx, *buf, sc);
1026 highlightCode(sc, sx, *buf, o);
1027 }
1028 buf.writestring("$(DDOC_OVERLOAD_SEPARATOR)");
1029 buf.writeByte(')');
1030 }
1031 buf.writestring(ddoc_decl_e);
1032 // Put the ddoc comment as the document 'description'
1033 buf.writestring(ddoc_decl_dd_s);
1034 {
1035 dc.writeSections(sc, &dc.a, buf);
1036 if (ScopeDsymbol sds = dc.a[0].isScopeDsymbol())
1037 emitMemberComments(sds, *buf, sc);
1038 }
1039 buf.writestring(ddoc_decl_dd_e);
1040 buf.writeByte(')');
1041 //printf("buf.2 = [[%.*s]]\n", cast(int)(buf.length - o0), buf.data + o0);
1042 }
1043 if (s)
1044 {
1045 DocComment* dc = DocComment.parse(s, com);
1046 dc.pmacrotable = &sc._module.macrotable;
1047 sc.lastdc = dc;
1048 }
1049 }
1050
1051 override void visit(Import imp)
1052 {
1053 if (imp.visible().kind != Visibility.Kind.public_ && sc.visibility.kind != Visibility.Kind.export_)
1054 return;
1055
1056 if (imp.comment)
1057 emit(sc, imp, imp.comment);
1058 }
1059
1060 override void visit(Declaration d)
1061 {
1062 //printf("Declaration::emitComment(%p '%s'), comment = '%s'\n", d, d.toChars(), d.comment);
1063 //printf("type = %p\n", d.type);
1064 const(char)* com = d.comment;
1065 if (TemplateDeclaration td = getEponymousParent(d))
1066 {
1067 if (isDitto(td.comment))
1068 com = td.comment;
1069 else
1070 com = Lexer.combineComments(td.comment.toDString(), com.toDString(), true);
1071 }
1072 else
1073 {
1074 if (!d.ident)
1075 return;
1076 if (!d.type)
1077 {
1078 if (!d.isCtorDeclaration() &&
1079 !d.isAliasDeclaration() &&
1080 !d.isVarDeclaration())
1081 {
1082 return;
1083 }
1084 }
1085 if (d.visibility.kind == Visibility.Kind.private_ || sc.visibility.kind == Visibility.Kind.private_)
1086 return;
1087 }
1088 if (!com)
1089 return;
1090 emit(sc, d, com);
1091 }
1092
1093 override void visit(AggregateDeclaration ad)
1094 {
1095 //printf("AggregateDeclaration::emitComment() '%s'\n", ad.toChars());
1096 const(char)* com = ad.comment;
1097 if (TemplateDeclaration td = getEponymousParent(ad))
1098 {
1099 if (isDitto(td.comment))
1100 com = td.comment;
1101 else
1102 com = Lexer.combineComments(td.comment.toDString(), com.toDString(), true);
1103 }
1104 else
1105 {
1106 if (ad.visible().kind == Visibility.Kind.private_ || sc.visibility.kind == Visibility.Kind.private_)
1107 return;
1108 if (!ad.comment)
1109 return;
1110 }
1111 if (!com)
1112 return;
1113 emit(sc, ad, com);
1114 }
1115
1116 override void visit(TemplateDeclaration td)
1117 {
1118 //printf("TemplateDeclaration::emitComment() '%s', kind = %s\n", td.toChars(), td.kind());
1119 if (td.visible().kind == Visibility.Kind.private_ || sc.visibility.kind == Visibility.Kind.private_)
1120 return;
1121 if (!td.comment)
1122 return;
1123 if (Dsymbol ss = getEponymousMember(td))
1124 {
1125 ss.accept(this);
1126 return;
1127 }
1128 emit(sc, td, td.comment);
1129 }
1130
1131 override void visit(EnumDeclaration ed)
1132 {
1133 if (ed.visible().kind == Visibility.Kind.private_ || sc.visibility.kind == Visibility.Kind.private_)
1134 return;
1135 if (ed.isAnonymous() && ed.members)
1136 {
1137 for (size_t i = 0; i < ed.members.dim; i++)
1138 {
1139 Dsymbol s = (*ed.members)[i];
1140 emitComment(s, *buf, sc);
1141 }
1142 return;
1143 }
1144 if (!ed.comment)
1145 return;
1146 if (ed.isAnonymous())
1147 return;
1148 emit(sc, ed, ed.comment);
1149 }
1150
1151 override void visit(EnumMember em)
1152 {
1153 //printf("EnumMember::emitComment(%p '%s'), comment = '%s'\n", em, em.toChars(), em.comment);
1154 if (em.visible().kind == Visibility.Kind.private_ || sc.visibility.kind == Visibility.Kind.private_)
1155 return;
1156 if (!em.comment)
1157 return;
1158 emit(sc, em, em.comment);
1159 }
1160
1161 override void visit(AttribDeclaration ad)
1162 {
1163 //printf("AttribDeclaration::emitComment(sc = %p)\n", sc);
1164 /* A general problem with this,
1165 * illustrated by https://issues.dlang.org/show_bug.cgi?id=2516
1166 * is that attributes are not transmitted through to the underlying
1167 * member declarations for template bodies, because semantic analysis
1168 * is not done for template declaration bodies
1169 * (only template instantiations).
1170 * Hence, Ddoc omits attributes from template members.
1171 */
1172 Dsymbols* d = ad.include(null);
1173 if (d)
1174 {
1175 for (size_t i = 0; i < d.dim; i++)
1176 {
1177 Dsymbol s = (*d)[i];
1178 //printf("AttribDeclaration::emitComment %s\n", s.toChars());
1179 emitComment(s, *buf, sc);
1180 }
1181 }
1182 }
1183
1184 override void visit(VisibilityDeclaration pd)
1185 {
1186 if (pd.decl)
1187 {
1188 Scope* scx = sc;
1189 sc = sc.copy();
1190 sc.visibility = pd.visibility;
1191 visit(cast(AttribDeclaration)pd);
1192 scx.lastdc = sc.lastdc;
1193 sc = sc.pop();
1194 }
1195 }
1196
1197 override void visit(ConditionalDeclaration cd)
1198 {
1199 //printf("ConditionalDeclaration::emitComment(sc = %p)\n", sc);
1200 if (cd.condition.inc != Include.notComputed)
1201 {
1202 visit(cast(AttribDeclaration)cd);
1203 return;
1204 }
1205 /* If generating doc comment, be careful because if we're inside
1206 * a template, then include(null) will fail.
1207 */
1208 Dsymbols* d = cd.decl ? cd.decl : cd.elsedecl;
1209 for (size_t i = 0; i < d.dim; i++)
1210 {
1211 Dsymbol s = (*d)[i];
1212 emitComment(s, *buf, sc);
1213 }
1214 }
1215 }
1216
1217 scope EmitComment v = new EmitComment(buf, sc);
1218 if (!s)
1219 v.emit(sc, null, null);
1220 else
1221 s.accept(v);
1222 }
1223
toDocBuffer(Dsymbol s,ref OutBuffer buf,Scope * sc)1224 private void toDocBuffer(Dsymbol s, ref OutBuffer buf, Scope* sc)
1225 {
1226 extern (C++) final class ToDocBuffer : Visitor
1227 {
1228 alias visit = Visitor.visit;
1229 public:
1230 OutBuffer* buf;
1231 Scope* sc;
1232
1233 extern (D) this(ref OutBuffer buf, Scope* sc)
1234 {
1235 this.buf = &buf;
1236 this.sc = sc;
1237 }
1238
1239 override void visit(Dsymbol s)
1240 {
1241 //printf("Dsymbol::toDocbuffer() %s\n", s.toChars());
1242 HdrGenState hgs;
1243 hgs.ddoc = true;
1244 .toCBuffer(s, buf, &hgs);
1245 }
1246
1247 void prefix(Dsymbol s)
1248 {
1249 if (s.isDeprecated())
1250 buf.writestring("deprecated ");
1251 if (Declaration d = s.isDeclaration())
1252 {
1253 emitVisibility(*buf, d);
1254 if (d.isStatic())
1255 buf.writestring("static ");
1256 else if (d.isFinal())
1257 buf.writestring("final ");
1258 else if (d.isAbstract())
1259 buf.writestring("abstract ");
1260
1261 if (d.isFuncDeclaration()) // functionToBufferFull handles this
1262 return;
1263
1264 if (d.isImmutable())
1265 buf.writestring("immutable ");
1266 if (d.storage_class & STC.shared_)
1267 buf.writestring("shared ");
1268 if (d.isWild())
1269 buf.writestring("inout ");
1270 if (d.isConst())
1271 buf.writestring("const ");
1272
1273 if (d.isSynchronized())
1274 buf.writestring("synchronized ");
1275
1276 if (d.storage_class & STC.manifest)
1277 buf.writestring("enum ");
1278
1279 // Add "auto" for the untyped variable in template members
1280 if (!d.type && d.isVarDeclaration() &&
1281 !d.isImmutable() && !(d.storage_class & STC.shared_) && !d.isWild() && !d.isConst() &&
1282 !d.isSynchronized())
1283 {
1284 buf.writestring("auto ");
1285 }
1286 }
1287 }
1288
1289 override void visit(Import i)
1290 {
1291 HdrGenState hgs;
1292 hgs.ddoc = true;
1293 emitVisibility(*buf, i);
1294 .toCBuffer(i, buf, &hgs);
1295 }
1296
1297 override void visit(Declaration d)
1298 {
1299 if (!d.ident)
1300 return;
1301 TemplateDeclaration td = getEponymousParent(d);
1302 //printf("Declaration::toDocbuffer() %s, originalType = %s, td = %s\n", d.toChars(), d.originalType ? d.originalType.toChars() : "--", td ? td.toChars() : "--");
1303 HdrGenState hgs;
1304 hgs.ddoc = true;
1305 if (d.isDeprecated())
1306 buf.writestring("$(DEPRECATED ");
1307 prefix(d);
1308 if (d.type)
1309 {
1310 Type origType = d.originalType ? d.originalType : d.type;
1311 if (origType.ty == Tfunction)
1312 {
1313 functionToBufferFull(cast(TypeFunction)origType, buf, d.ident, &hgs, td);
1314 }
1315 else
1316 .toCBuffer(origType, buf, d.ident, &hgs);
1317 }
1318 else
1319 buf.writestring(d.ident.toString());
1320 if (d.isVarDeclaration() && td)
1321 {
1322 buf.writeByte('(');
1323 if (td.origParameters && td.origParameters.dim)
1324 {
1325 for (size_t i = 0; i < td.origParameters.dim; i++)
1326 {
1327 if (i)
1328 buf.writestring(", ");
1329 toCBuffer((*td.origParameters)[i], buf, &hgs);
1330 }
1331 }
1332 buf.writeByte(')');
1333 }
1334 // emit constraints if declaration is a templated declaration
1335 if (td && td.constraint)
1336 {
1337 bool noFuncDecl = td.isFuncDeclaration() is null;
1338 if (noFuncDecl)
1339 {
1340 buf.writestring("$(DDOC_CONSTRAINT ");
1341 }
1342
1343 .toCBuffer(td.constraint, buf, &hgs);
1344
1345 if (noFuncDecl)
1346 {
1347 buf.writestring(")");
1348 }
1349 }
1350 if (d.isDeprecated())
1351 buf.writestring(")");
1352 buf.writestring(";\n");
1353 }
1354
1355 override void visit(AliasDeclaration ad)
1356 {
1357 //printf("AliasDeclaration::toDocbuffer() %s\n", ad.toChars());
1358 if (!ad.ident)
1359 return;
1360 if (ad.isDeprecated())
1361 buf.writestring("deprecated ");
1362 emitVisibility(*buf, ad);
1363 buf.printf("alias %s = ", ad.toChars());
1364 if (Dsymbol s = ad.aliassym) // ident alias
1365 {
1366 prettyPrintDsymbol(s, ad.parent);
1367 }
1368 else if (Type type = ad.getType()) // type alias
1369 {
1370 if (type.ty == Tclass || type.ty == Tstruct || type.ty == Tenum)
1371 {
1372 if (Dsymbol s = type.toDsymbol(null)) // elaborate type
1373 prettyPrintDsymbol(s, ad.parent);
1374 else
1375 buf.writestring(type.toChars());
1376 }
1377 else
1378 {
1379 // simple type
1380 buf.writestring(type.toChars());
1381 }
1382 }
1383 buf.writestring(";\n");
1384 }
1385
1386 void parentToBuffer(Dsymbol s)
1387 {
1388 if (s && !s.isPackage() && !s.isModule())
1389 {
1390 parentToBuffer(s.parent);
1391 buf.writestring(s.toChars());
1392 buf.writestring(".");
1393 }
1394 }
1395
1396 static bool inSameModule(Dsymbol s, Dsymbol p)
1397 {
1398 for (; s; s = s.parent)
1399 {
1400 if (s.isModule())
1401 break;
1402 }
1403 for (; p; p = p.parent)
1404 {
1405 if (p.isModule())
1406 break;
1407 }
1408 return s == p;
1409 }
1410
1411 void prettyPrintDsymbol(Dsymbol s, Dsymbol parent)
1412 {
1413 if (s.parent && (s.parent == parent)) // in current scope -> naked name
1414 {
1415 buf.writestring(s.toChars());
1416 }
1417 else if (!inSameModule(s, parent)) // in another module -> full name
1418 {
1419 buf.writestring(s.toPrettyChars());
1420 }
1421 else // nested in a type in this module -> full name w/o module name
1422 {
1423 // if alias is nested in a user-type use module-scope lookup
1424 if (!parent.isModule() && !parent.isPackage())
1425 buf.writestring(".");
1426 parentToBuffer(s.parent);
1427 buf.writestring(s.toChars());
1428 }
1429 }
1430
1431 override void visit(AggregateDeclaration ad)
1432 {
1433 if (!ad.ident)
1434 return;
1435 version (none)
1436 {
1437 emitVisibility(buf, ad);
1438 }
1439 buf.printf("%s %s", ad.kind(), ad.toChars());
1440 buf.writestring(";\n");
1441 }
1442
1443 override void visit(StructDeclaration sd)
1444 {
1445 //printf("StructDeclaration::toDocbuffer() %s\n", sd.toChars());
1446 if (!sd.ident)
1447 return;
1448 version (none)
1449 {
1450 emitVisibility(buf, sd);
1451 }
1452 if (TemplateDeclaration td = getEponymousParent(sd))
1453 {
1454 toDocBuffer(td, *buf, sc);
1455 }
1456 else
1457 {
1458 buf.printf("%s %s", sd.kind(), sd.toChars());
1459 }
1460 buf.writestring(";\n");
1461 }
1462
1463 override void visit(ClassDeclaration cd)
1464 {
1465 //printf("ClassDeclaration::toDocbuffer() %s\n", cd.toChars());
1466 if (!cd.ident)
1467 return;
1468 version (none)
1469 {
1470 emitVisibility(*buf, cd);
1471 }
1472 if (TemplateDeclaration td = getEponymousParent(cd))
1473 {
1474 toDocBuffer(td, *buf, sc);
1475 }
1476 else
1477 {
1478 if (!cd.isInterfaceDeclaration() && cd.isAbstract())
1479 buf.writestring("abstract ");
1480 buf.printf("%s %s", cd.kind(), cd.toChars());
1481 }
1482 int any = 0;
1483 for (size_t i = 0; i < cd.baseclasses.dim; i++)
1484 {
1485 BaseClass* bc = (*cd.baseclasses)[i];
1486 if (bc.sym && bc.sym.ident == Id.Object)
1487 continue;
1488 if (any)
1489 buf.writestring(", ");
1490 else
1491 {
1492 buf.writestring(": ");
1493 any = 1;
1494 }
1495
1496 if (bc.sym)
1497 {
1498 buf.printf("$(DDOC_PSUPER_SYMBOL %s)", bc.sym.toPrettyChars());
1499 }
1500 else
1501 {
1502 HdrGenState hgs;
1503 .toCBuffer(bc.type, buf, null, &hgs);
1504 }
1505 }
1506 buf.writestring(";\n");
1507 }
1508
1509 override void visit(EnumDeclaration ed)
1510 {
1511 if (!ed.ident)
1512 return;
1513 buf.printf("%s %s", ed.kind(), ed.toChars());
1514 if (ed.memtype)
1515 {
1516 buf.writestring(": $(DDOC_ENUM_BASETYPE ");
1517 HdrGenState hgs;
1518 .toCBuffer(ed.memtype, buf, null, &hgs);
1519 buf.writestring(")");
1520 }
1521 buf.writestring(";\n");
1522 }
1523
1524 override void visit(EnumMember em)
1525 {
1526 if (!em.ident)
1527 return;
1528 buf.writestring(em.toChars());
1529 }
1530 }
1531
1532 scope ToDocBuffer v = new ToDocBuffer(buf, sc);
1533 s.accept(v);
1534 }
1535
1536 /***********************************************************
1537 */
1538 struct DocComment
1539 {
1540 Sections sections; // Section*[]
1541 Section summary;
1542 Section copyright;
1543 Section macros;
1544 MacroTable* pmacrotable;
1545 Escape* escapetable;
1546 Dsymbols a;
1547
parseDocComment1548 static DocComment* parse(Dsymbol s, const(char)* comment)
1549 {
1550 //printf("parse(%s): '%s'\n", s.toChars(), comment);
1551 auto dc = new DocComment();
1552 dc.a.push(s);
1553 if (!comment)
1554 return dc;
1555 dc.parseSections(comment);
1556 for (size_t i = 0; i < dc.sections.dim; i++)
1557 {
1558 Section sec = dc.sections[i];
1559 if (iequals("copyright", sec.name))
1560 {
1561 dc.copyright = sec;
1562 }
1563 if (iequals("macros", sec.name))
1564 {
1565 dc.macros = sec;
1566 }
1567 }
1568 return dc;
1569 }
1570
1571 /************************************************
1572 * Parse macros out of Macros: section.
1573 * Macros are of the form:
1574 * name1 = value1
1575 *
1576 * name2 = value2
1577 */
parseMacrosDocComment1578 extern(D) static void parseMacros(
1579 Escape* escapetable, ref MacroTable pmacrotable, const(char)[] m)
1580 {
1581 const(char)* p = m.ptr;
1582 size_t len = m.length;
1583 const(char)* pend = p + len;
1584 const(char)* tempstart = null;
1585 size_t templen = 0;
1586 const(char)* namestart = null;
1587 size_t namelen = 0; // !=0 if line continuation
1588 const(char)* textstart = null;
1589 size_t textlen = 0;
1590 while (p < pend)
1591 {
1592 // Skip to start of macro
1593 while (1)
1594 {
1595 if (p >= pend)
1596 goto Ldone;
1597 switch (*p)
1598 {
1599 case ' ':
1600 case '\t':
1601 p++;
1602 continue;
1603 case '\r':
1604 case '\n':
1605 p++;
1606 goto Lcont;
1607 default:
1608 if (isIdStart(p))
1609 break;
1610 if (namelen)
1611 goto Ltext; // continuation of prev macro
1612 goto Lskipline;
1613 }
1614 break;
1615 }
1616 tempstart = p;
1617 while (1)
1618 {
1619 if (p >= pend)
1620 goto Ldone;
1621 if (!isIdTail(p))
1622 break;
1623 p += utfStride(p);
1624 }
1625 templen = p - tempstart;
1626 while (1)
1627 {
1628 if (p >= pend)
1629 goto Ldone;
1630 if (!(*p == ' ' || *p == '\t'))
1631 break;
1632 p++;
1633 }
1634 if (*p != '=')
1635 {
1636 if (namelen)
1637 goto Ltext; // continuation of prev macro
1638 goto Lskipline;
1639 }
1640 p++;
1641 if (p >= pend)
1642 goto Ldone;
1643 if (namelen)
1644 {
1645 // Output existing macro
1646 L1:
1647 //printf("macro '%.*s' = '%.*s'\n", cast(int)namelen, namestart, cast(int)textlen, textstart);
1648 if (iequals("ESCAPES", namestart[0 .. namelen]))
1649 parseEscapes(escapetable, textstart[0 .. textlen]);
1650 else
1651 pmacrotable.define(namestart[0 .. namelen], textstart[0 .. textlen]);
1652 namelen = 0;
1653 if (p >= pend)
1654 break;
1655 }
1656 namestart = tempstart;
1657 namelen = templen;
1658 while (p < pend && (*p == ' ' || *p == '\t'))
1659 p++;
1660 textstart = p;
1661 Ltext:
1662 while (p < pend && *p != '\r' && *p != '\n')
1663 p++;
1664 textlen = p - textstart;
1665 p++;
1666 //printf("p = %p, pend = %p\n", p, pend);
1667 Lcont:
1668 continue;
1669 Lskipline:
1670 // Ignore this line
1671 while (p < pend && *p != '\r' && *p != '\n')
1672 p++;
1673 }
1674 Ldone:
1675 if (namelen)
1676 goto L1; // write out last one
1677 }
1678
1679 /**************************************
1680 * Parse escapes of the form:
1681 * /c/string/
1682 * where c is a single character.
1683 * Multiple escapes can be separated
1684 * by whitespace and/or commas.
1685 */
parseEscapesDocComment1686 static void parseEscapes(Escape* escapetable, const(char)[] text)
1687 {
1688 if (!escapetable)
1689 {
1690 escapetable = new Escape();
1691 memset(escapetable, 0, Escape.sizeof);
1692 }
1693 //printf("parseEscapes('%.*s') pescapetable = %p\n", cast(int)text.length, text.ptr, escapetable);
1694 const(char)* p = text.ptr;
1695 const(char)* pend = p + text.length;
1696 while (1)
1697 {
1698 while (1)
1699 {
1700 if (p + 4 >= pend)
1701 return;
1702 if (!(*p == ' ' || *p == '\t' || *p == '\r' || *p == '\n' || *p == ','))
1703 break;
1704 p++;
1705 }
1706 if (p[0] != '/' || p[2] != '/')
1707 return;
1708 char c = p[1];
1709 p += 3;
1710 const(char)* start = p;
1711 while (1)
1712 {
1713 if (p >= pend)
1714 return;
1715 if (*p == '/')
1716 break;
1717 p++;
1718 }
1719 size_t len = p - start;
1720 char* s = cast(char*)memcpy(mem.xmalloc(len + 1), start, len);
1721 s[len] = 0;
1722 escapetable.strings[c] = s[0 .. len];
1723 //printf("\t%c = '%s'\n", c, s);
1724 p++;
1725 }
1726 }
1727
1728 /*****************************************
1729 * Parse next paragraph out of *pcomment.
1730 * Update *pcomment to point past paragraph.
1731 * Returns NULL if no more paragraphs.
1732 * If paragraph ends in 'identifier:',
1733 * then (*pcomment)[0 .. idlen] is the identifier.
1734 */
parseSectionsDocComment1735 void parseSections(const(char)* comment)
1736 {
1737 const(char)* p;
1738 const(char)* pstart;
1739 const(char)* pend;
1740 const(char)* idstart = null; // dead-store to prevent spurious warning
1741 size_t idlen;
1742 const(char)* name = null;
1743 size_t namelen = 0;
1744 //printf("parseSections('%s')\n", comment);
1745 p = comment;
1746 while (*p)
1747 {
1748 const(char)* pstart0 = p;
1749 p = skipwhitespace(p);
1750 pstart = p;
1751 pend = p;
1752
1753 // Undo indent if starting with a list item
1754 if ((*p == '-' || *p == '+' || *p == '*') && (*(p+1) == ' ' || *(p+1) == '\t'))
1755 pstart = pstart0;
1756 else
1757 {
1758 const(char)* pitem = p;
1759 while (*pitem >= '0' && *pitem <= '9')
1760 ++pitem;
1761 if (pitem > p && *pitem == '.' && (*(pitem+1) == ' ' || *(pitem+1) == '\t'))
1762 pstart = pstart0;
1763 }
1764
1765 /* Find end of section, which is ended by one of:
1766 * 'identifier:' (but not inside a code section)
1767 * '\0'
1768 */
1769 idlen = 0;
1770 int inCode = 0;
1771 while (1)
1772 {
1773 // Check for start/end of a code section
1774 if (*p == '-' || *p == '`' || *p == '~')
1775 {
1776 char c = *p;
1777 int numdash = 0;
1778 while (*p == c)
1779 {
1780 ++numdash;
1781 p++;
1782 }
1783 // BUG: handle UTF PS and LS too
1784 if ((!*p || *p == '\r' || *p == '\n' || (!inCode && c != '-')) && numdash >= 3)
1785 {
1786 inCode = inCode == c ? false : c;
1787 if (inCode)
1788 {
1789 // restore leading indentation
1790 while (pstart0 < pstart && isIndentWS(pstart - 1))
1791 --pstart;
1792 }
1793 }
1794 pend = p;
1795 }
1796 if (!inCode && isIdStart(p))
1797 {
1798 const(char)* q = p + utfStride(p);
1799 while (isIdTail(q))
1800 q += utfStride(q);
1801
1802 // Detected tag ends it
1803 if (*q == ':' && isupper(*p)
1804 && (isspace(q[1]) || q[1] == 0))
1805 {
1806 idlen = q - p;
1807 idstart = p;
1808 for (pend = p; pend > pstart; pend--)
1809 {
1810 if (pend[-1] == '\n')
1811 break;
1812 }
1813 p = q + 1;
1814 break;
1815 }
1816 }
1817 while (1)
1818 {
1819 if (!*p)
1820 goto L1;
1821 if (*p == '\n')
1822 {
1823 p++;
1824 if (*p == '\n' && !summary && !namelen && !inCode)
1825 {
1826 pend = p;
1827 p++;
1828 goto L1;
1829 }
1830 break;
1831 }
1832 p++;
1833 pend = p;
1834 }
1835 p = skipwhitespace(p);
1836 }
1837 L1:
1838 if (namelen || pstart < pend)
1839 {
1840 Section s;
1841 if (iequals("Params", name[0 .. namelen]))
1842 s = new ParamSection();
1843 else if (iequals("Macros", name[0 .. namelen]))
1844 s = new MacroSection();
1845 else
1846 s = new Section();
1847 s.name = name[0 .. namelen];
1848 s.body_ = pstart[0 .. pend - pstart];
1849 s.nooutput = 0;
1850 //printf("Section: '%.*s' = '%.*s'\n", cast(int)s.namelen, s.name, cast(int)s.bodylen, s.body);
1851 sections.push(s);
1852 if (!summary && !namelen)
1853 summary = s;
1854 }
1855 if (idlen)
1856 {
1857 name = idstart;
1858 namelen = idlen;
1859 }
1860 else
1861 {
1862 name = null;
1863 namelen = 0;
1864 if (!*p)
1865 break;
1866 }
1867 }
1868 }
1869
writeSectionsDocComment1870 void writeSections(Scope* sc, Dsymbols* a, OutBuffer* buf)
1871 {
1872 assert(a.dim);
1873 //printf("DocComment::writeSections()\n");
1874 Loc loc = (*a)[0].loc;
1875 if (Module m = (*a)[0].isModule())
1876 {
1877 if (m.md)
1878 loc = m.md.loc;
1879 }
1880 size_t offset1 = buf.length;
1881 buf.writestring("$(DDOC_SECTIONS ");
1882 size_t offset2 = buf.length;
1883 for (size_t i = 0; i < sections.dim; i++)
1884 {
1885 Section sec = sections[i];
1886 if (sec.nooutput)
1887 continue;
1888 //printf("Section: '%.*s' = '%.*s'\n", cast(int)sec.namelen, sec.name, cast(int)sec.bodylen, sec.body);
1889 if (!sec.name.length && i == 0)
1890 {
1891 buf.writestring("$(DDOC_SUMMARY ");
1892 size_t o = buf.length;
1893 buf.write(sec.body_);
1894 escapeStrayParenthesis(loc, buf, o, true);
1895 highlightText(sc, a, loc, *buf, o);
1896 buf.writestring(")");
1897 }
1898 else
1899 sec.write(loc, &this, sc, a, buf);
1900 }
1901 for (size_t i = 0; i < a.dim; i++)
1902 {
1903 Dsymbol s = (*a)[i];
1904 if (Dsymbol td = getEponymousParent(s))
1905 s = td;
1906 for (UnitTestDeclaration utd = s.ddocUnittest; utd; utd = utd.ddocUnittest)
1907 {
1908 if (utd.visibility.kind == Visibility.Kind.private_ || !utd.comment || !utd.fbody)
1909 continue;
1910 // Strip whitespaces to avoid showing empty summary
1911 const(char)* c = utd.comment;
1912 while (*c == ' ' || *c == '\t' || *c == '\n' || *c == '\r')
1913 ++c;
1914 buf.writestring("$(DDOC_EXAMPLES ");
1915 size_t o = buf.length;
1916 buf.writestring(cast(char*)c);
1917 if (utd.codedoc)
1918 {
1919 auto codedoc = utd.codedoc.stripLeadingNewlines;
1920 size_t n = getCodeIndent(codedoc);
1921 while (n--)
1922 buf.writeByte(' ');
1923 buf.writestring("----\n");
1924 buf.writestring(codedoc);
1925 buf.writestring("----\n");
1926 highlightText(sc, a, loc, *buf, o);
1927 }
1928 buf.writestring(")");
1929 }
1930 }
1931 if (buf.length == offset2)
1932 {
1933 /* Didn't write out any sections, so back out last write
1934 */
1935 buf.setsize(offset1);
1936 buf.writestring("\n");
1937 }
1938 else
1939 buf.writestring(")");
1940 }
1941 }
1942
1943 /*****************************************
1944 * Return true if comment consists entirely of "ditto".
1945 */
isDitto(const (char)* comment)1946 private bool isDitto(const(char)* comment)
1947 {
1948 if (comment)
1949 {
1950 const(char)* p = skipwhitespace(comment);
1951 if (Port.memicmp(p, "ditto", 5) == 0 && *skipwhitespace(p + 5) == 0)
1952 return true;
1953 }
1954 return false;
1955 }
1956
1957 /**********************************************
1958 * Skip white space.
1959 */
skipwhitespace(const (char)* p)1960 private const(char)* skipwhitespace(const(char)* p)
1961 {
1962 return skipwhitespace(p.toDString).ptr;
1963 }
1964
1965 /// Ditto
skipwhitespace(const (char)[]p)1966 private const(char)[] skipwhitespace(const(char)[] p)
1967 {
1968 foreach (idx, char c; p)
1969 {
1970 switch (c)
1971 {
1972 case ' ':
1973 case '\t':
1974 case '\n':
1975 continue;
1976 default:
1977 return p[idx .. $];
1978 }
1979 }
1980 return p[$ .. $];
1981 }
1982
1983 /************************************************
1984 * Scan past all instances of the given characters.
1985 * Params:
1986 * buf = an OutBuffer containing the DDoc
1987 * i = the index within `buf` to start scanning from
1988 * chars = the characters to skip; order is unimportant
1989 * Returns: the index after skipping characters.
1990 */
skipChars(ref OutBuffer buf,size_t i,string chars)1991 private size_t skipChars(ref OutBuffer buf, size_t i, string chars)
1992 {
1993 Outer:
1994 foreach (j, c; buf[][i..$])
1995 {
1996 foreach (d; chars)
1997 {
1998 if (d == c)
1999 continue Outer;
2000 }
2001 return i + j;
2002 }
2003 return buf.length;
2004 }
2005
2006 unittest {
2007 OutBuffer buf;
2008 string data = "test ---\r\n\r\nend";
2009 buf.write(data);
2010
2011 assert(skipChars(buf, 0, "-") == 0);
2012 assert(skipChars(buf, 4, "-") == 4);
2013 assert(skipChars(buf, 4, " -") == 8);
2014 assert(skipChars(buf, 8, "\r\n") == 12);
2015 assert(skipChars(buf, 12, "dne") == 15);
2016 }
2017
2018 /****************************************************
2019 * Replace all instances of `c` with `r` in the given string
2020 * Params:
2021 * s = the string to do replacements in
2022 * c = the character to look for
2023 * r = the string to replace `c` with
2024 * Returns: `s` with `c` replaced with `r`
2025 */
inout(char)2026 private inout(char)[] replaceChar(inout(char)[] s, char c, string r) pure
2027 {
2028 int count = 0;
2029 foreach (char sc; s)
2030 if (sc == c)
2031 ++count;
2032 if (count == 0)
2033 return s;
2034
2035 char[] result;
2036 result.reserve(s.length - count + (r.length * count));
2037 size_t start = 0;
2038 foreach (i, char sc; s)
2039 {
2040 if (sc == c)
2041 {
2042 result ~= s[start..i];
2043 result ~= r;
2044 start = i+1;
2045 }
2046 }
2047 result ~= s[start..$];
2048 return result;
2049 }
2050
2051 ///
2052 unittest
2053 {
2054 assert("".replaceChar(',', "$(COMMA)") == "");
2055 assert("ab".replaceChar(',', "$(COMMA)") == "ab");
2056 assert("a,b".replaceChar(',', "$(COMMA)") == "a$(COMMA)b");
2057 assert("a,,b".replaceChar(',', "$(COMMA)") == "a$(COMMA)$(COMMA)b");
2058 assert(",ab".replaceChar(',', "$(COMMA)") == "$(COMMA)ab");
2059 assert("ab,".replaceChar(',', "$(COMMA)") == "ab$(COMMA)");
2060 }
2061
2062 /**
2063 * Return a lowercased copy of a string.
2064 * Params:
2065 * s = the string to lowercase
2066 * Returns: the lowercase version of the string or the original if already lowercase
2067 */
toLowercase(string s)2068 private string toLowercase(string s) pure
2069 {
2070 string lower;
2071 foreach (size_t i; 0..s.length)
2072 {
2073 char c = s[i];
2074 // TODO: maybe unicode lowercase, somehow
2075 if (c >= 'A' && c <= 'Z')
2076 {
2077 if (!lower.length) {
2078 lower.reserve(s.length);
2079 }
2080 lower ~= s[lower.length..i];
2081 c += 'a' - 'A';
2082 lower ~= c;
2083 }
2084 }
2085 if (lower.length)
2086 lower ~= s[lower.length..$];
2087 else
2088 lower = s;
2089 return lower;
2090 }
2091
2092 ///
2093 unittest
2094 {
2095 assert("".toLowercase == "");
2096 assert("abc".toLowercase == "abc");
2097 assert("ABC".toLowercase == "abc");
2098 assert("aBc".toLowercase == "abc");
2099 }
2100
2101 /************************************************
2102 * Get the indent from one index to another, counting tab stops as four spaces wide
2103 * per the Markdown spec.
2104 * Params:
2105 * buf = an OutBuffer containing the DDoc
2106 * from = the index within `buf` to start counting from, inclusive
2107 * to = the index within `buf` to stop counting at, exclusive
2108 * Returns: the indent
2109 */
getMarkdownIndent(ref OutBuffer buf,size_t from,size_t to)2110 private int getMarkdownIndent(ref OutBuffer buf, size_t from, size_t to)
2111 {
2112 const slice = buf[];
2113 if (to > slice.length)
2114 to = slice.length;
2115 int indent = 0;
2116 foreach (const c; slice[from..to])
2117 indent += (c == '\t') ? 4 - (indent % 4) : 1;
2118 return indent;
2119 }
2120
2121 /************************************************
2122 * Scan forward to one of:
2123 * start of identifier
2124 * beginning of next line
2125 * end of buf
2126 */
skiptoident(ref OutBuffer buf,size_t i)2127 size_t skiptoident(ref OutBuffer buf, size_t i)
2128 {
2129 const slice = buf[];
2130 while (i < slice.length)
2131 {
2132 dchar c;
2133 size_t oi = i;
2134 if (utf_decodeChar(slice, i, c))
2135 {
2136 /* Ignore UTF errors, but still consume input
2137 */
2138 break;
2139 }
2140 if (c >= 0x80)
2141 {
2142 if (!isUniAlpha(c))
2143 continue;
2144 }
2145 else if (!(isalpha(c) || c == '_' || c == '\n'))
2146 continue;
2147 i = oi;
2148 break;
2149 }
2150 return i;
2151 }
2152
2153 /************************************************
2154 * Scan forward past end of identifier.
2155 */
skippastident(ref OutBuffer buf,size_t i)2156 private size_t skippastident(ref OutBuffer buf, size_t i)
2157 {
2158 const slice = buf[];
2159 while (i < slice.length)
2160 {
2161 dchar c;
2162 size_t oi = i;
2163 if (utf_decodeChar(slice, i, c))
2164 {
2165 /* Ignore UTF errors, but still consume input
2166 */
2167 break;
2168 }
2169 if (c >= 0x80)
2170 {
2171 if (isUniAlpha(c))
2172 continue;
2173 }
2174 else if (isalnum(c) || c == '_')
2175 continue;
2176 i = oi;
2177 break;
2178 }
2179 return i;
2180 }
2181
2182 /************************************************
2183 * Scan forward past end of an identifier that might
2184 * contain dots (e.g. `abc.def`)
2185 */
skipPastIdentWithDots(ref OutBuffer buf,size_t i)2186 private size_t skipPastIdentWithDots(ref OutBuffer buf, size_t i)
2187 {
2188 const slice = buf[];
2189 bool lastCharWasDot;
2190 while (i < slice.length)
2191 {
2192 dchar c;
2193 size_t oi = i;
2194 if (utf_decodeChar(slice, i, c))
2195 {
2196 /* Ignore UTF errors, but still consume input
2197 */
2198 break;
2199 }
2200 if (c == '.')
2201 {
2202 // We need to distinguish between `abc.def`, abc..def`, and `abc.`
2203 // Only `abc.def` is a valid identifier
2204
2205 if (lastCharWasDot)
2206 {
2207 i = oi;
2208 break;
2209 }
2210
2211 lastCharWasDot = true;
2212 continue;
2213 }
2214 else
2215 {
2216 if (c >= 0x80)
2217 {
2218 if (isUniAlpha(c))
2219 {
2220 lastCharWasDot = false;
2221 continue;
2222 }
2223 }
2224 else if (isalnum(c) || c == '_')
2225 {
2226 lastCharWasDot = false;
2227 continue;
2228 }
2229 i = oi;
2230 break;
2231 }
2232 }
2233
2234 // if `abc.`
2235 if (lastCharWasDot)
2236 return i - 1;
2237
2238 return i;
2239 }
2240
2241 /************************************************
2242 * Scan forward past URL starting at i.
2243 * We don't want to highlight parts of a URL.
2244 * Returns:
2245 * i if not a URL
2246 * index just past it if it is a URL
2247 */
skippastURL(ref OutBuffer buf,size_t i)2248 private size_t skippastURL(ref OutBuffer buf, size_t i)
2249 {
2250 const slice = buf[][i .. $];
2251 size_t j;
2252 bool sawdot = false;
2253 if (slice.length > 7 && Port.memicmp(slice.ptr, "http://", 7) == 0)
2254 {
2255 j = 7;
2256 }
2257 else if (slice.length > 8 && Port.memicmp(slice.ptr, "https://", 8) == 0)
2258 {
2259 j = 8;
2260 }
2261 else
2262 goto Lno;
2263 for (; j < slice.length; j++)
2264 {
2265 const c = slice[j];
2266 if (isalnum(c))
2267 continue;
2268 if (c == '-' || c == '_' || c == '?' || c == '=' || c == '%' ||
2269 c == '&' || c == '/' || c == '+' || c == '#' || c == '~')
2270 continue;
2271 if (c == '.')
2272 {
2273 sawdot = true;
2274 continue;
2275 }
2276 break;
2277 }
2278 if (sawdot)
2279 return i + j;
2280 Lno:
2281 return i;
2282 }
2283
2284 /****************************************************
2285 * Remove a previously-inserted blank line macro.
2286 * Params:
2287 * buf = an OutBuffer containing the DDoc
2288 * iAt = the index within `buf` of the start of the `$(DDOC_BLANKLINE)`
2289 * macro. Upon function return its value is set to `0`.
2290 * i = an index within `buf`. If `i` is after `iAt` then it gets
2291 * reduced by the length of the removed macro.
2292 */
removeBlankLineMacro(ref OutBuffer buf,ref size_t iAt,ref size_t i)2293 private void removeBlankLineMacro(ref OutBuffer buf, ref size_t iAt, ref size_t i)
2294 {
2295 if (!iAt)
2296 return;
2297
2298 enum macroLength = "$(DDOC_BLANKLINE)".length;
2299 buf.remove(iAt, macroLength);
2300 if (i > iAt)
2301 i -= macroLength;
2302 iAt = 0;
2303 }
2304
2305 /****************************************************
2306 * Attempt to detect and replace a Markdown thematic break (HR). These are three
2307 * or more of the same delimiter, optionally with spaces or tabs between any of
2308 * them, e.g. `\n- - -\n` becomes `\n$(HR)\n`
2309 * Params:
2310 * buf = an OutBuffer containing the DDoc
2311 * i = the index within `buf` of the first character of a potential
2312 * thematic break. If the replacement is made `i` changes to
2313 * point to the closing parenthesis of the `$(HR)` macro.
2314 * iLineStart = the index within `buf` that the thematic break's line starts at
2315 * loc = the current location within the file
2316 * Returns: whether a thematic break was replaced
2317 */
replaceMarkdownThematicBreak(ref OutBuffer buf,ref size_t i,size_t iLineStart,const ref Loc loc)2318 private bool replaceMarkdownThematicBreak(ref OutBuffer buf, ref size_t i, size_t iLineStart, const ref Loc loc)
2319 {
2320 if (!global.params.markdown)
2321 return false;
2322
2323 const slice = buf[];
2324 const c = buf[i];
2325 size_t j = i + 1;
2326 int repeat = 1;
2327 for (; j < slice.length; j++)
2328 {
2329 if (buf[j] == c)
2330 ++repeat;
2331 else if (buf[j] != ' ' && buf[j] != '\t')
2332 break;
2333 }
2334 if (repeat >= 3)
2335 {
2336 if (j >= buf.length || buf[j] == '\n' || buf[j] == '\r')
2337 {
2338 if (global.params.vmarkdown)
2339 {
2340 const s = buf[][i..j];
2341 message(loc, "Ddoc: converted '%.*s' to a thematic break", cast(int)s.length, s.ptr);
2342 }
2343
2344 buf.remove(iLineStart, j - iLineStart);
2345 i = buf.insert(iLineStart, "$(HR)") - 1;
2346 return true;
2347 }
2348 }
2349 return false;
2350 }
2351
2352 /****************************************************
2353 * Detect the level of an ATX-style heading, e.g. `## This is a heading` would
2354 * have a level of `2`.
2355 * Params:
2356 * buf = an OutBuffer containing the DDoc
2357 * i = the index within `buf` of the first `#` character
2358 * Returns:
2359 * the detected heading level from 1 to 6, or
2360 * 0 if not at an ATX heading
2361 */
detectAtxHeadingLevel(ref OutBuffer buf,const size_t i)2362 private int detectAtxHeadingLevel(ref OutBuffer buf, const size_t i)
2363 {
2364 if (!global.params.markdown)
2365 return 0;
2366
2367 const iHeadingStart = i;
2368 const iAfterHashes = skipChars(buf, i, "#");
2369 const headingLevel = cast(int) (iAfterHashes - iHeadingStart);
2370 if (headingLevel > 6)
2371 return 0;
2372
2373 const iTextStart = skipChars(buf, iAfterHashes, " \t");
2374 const emptyHeading = buf[iTextStart] == '\r' || buf[iTextStart] == '\n';
2375
2376 // require whitespace
2377 if (!emptyHeading && iTextStart == iAfterHashes)
2378 return 0;
2379
2380 return headingLevel;
2381 }
2382
2383 /****************************************************
2384 * Remove any trailing `##` suffix from an ATX-style heading.
2385 * Params:
2386 * buf = an OutBuffer containing the DDoc
2387 * i = the index within `buf` to start looking for a suffix at
2388 */
removeAnyAtxHeadingSuffix(ref OutBuffer buf,size_t i)2389 private void removeAnyAtxHeadingSuffix(ref OutBuffer buf, size_t i)
2390 {
2391 size_t j = i;
2392 size_t iSuffixStart = 0;
2393 size_t iWhitespaceStart = j;
2394 const slice = buf[];
2395 for (; j < slice.length; j++)
2396 {
2397 switch (slice[j])
2398 {
2399 case '#':
2400 if (iWhitespaceStart && !iSuffixStart)
2401 iSuffixStart = j;
2402 continue;
2403 case ' ':
2404 case '\t':
2405 if (!iWhitespaceStart)
2406 iWhitespaceStart = j;
2407 continue;
2408 case '\r':
2409 case '\n':
2410 break;
2411 default:
2412 iSuffixStart = 0;
2413 iWhitespaceStart = 0;
2414 continue;
2415 }
2416 break;
2417 }
2418 if (iSuffixStart)
2419 buf.remove(iWhitespaceStart, j - iWhitespaceStart);
2420 }
2421
2422 /****************************************************
2423 * Wrap text in a Markdown heading macro, e.g. `$(H2 heading text`).
2424 * Params:
2425 * buf = an OutBuffer containing the DDoc
2426 * iStart = the index within `buf` that the Markdown heading starts at
2427 * iEnd = the index within `buf` of the character after the last
2428 * heading character. Is incremented by the length of the
2429 * inserted heading macro when this function ends.
2430 * loc = the location of the Ddoc within the file
2431 * headingLevel = the level (1-6) of heading to end. Is set to `0` when this
2432 * function ends.
2433 */
endMarkdownHeading(ref OutBuffer buf,size_t iStart,ref size_t iEnd,const ref Loc loc,ref int headingLevel)2434 private void endMarkdownHeading(ref OutBuffer buf, size_t iStart, ref size_t iEnd, const ref Loc loc, ref int headingLevel)
2435 {
2436 if (!global.params.markdown)
2437 return;
2438 if (global.params.vmarkdown)
2439 {
2440 const s = buf[][iStart..iEnd];
2441 message(loc, "Ddoc: added heading '%.*s'", cast(int)s.length, s.ptr);
2442 }
2443
2444 char[5] heading = "$(H0 ";
2445 heading[3] = cast(char) ('0' + headingLevel);
2446 buf.insert(iStart, heading);
2447 iEnd += 5;
2448 size_t iBeforeNewline = iEnd;
2449 while (buf[iBeforeNewline-1] == '\r' || buf[iBeforeNewline-1] == '\n')
2450 --iBeforeNewline;
2451 buf.insert(iBeforeNewline, ")");
2452 headingLevel = 0;
2453 }
2454
2455 /****************************************************
2456 * End all nested Markdown quotes, if inside any.
2457 * Params:
2458 * buf = an OutBuffer containing the DDoc
2459 * i = the index within `buf` of the character after the quote text.
2460 * quoteLevel = the current quote level. Is set to `0` when this function ends.
2461 * Returns: the amount that `i` was moved
2462 */
endAllMarkdownQuotes(ref OutBuffer buf,size_t i,ref int quoteLevel)2463 private size_t endAllMarkdownQuotes(ref OutBuffer buf, size_t i, ref int quoteLevel)
2464 {
2465 const length = quoteLevel;
2466 for (; quoteLevel > 0; --quoteLevel)
2467 i = buf.insert(i, ")");
2468 return length;
2469 }
2470
2471 /****************************************************
2472 * Convenience function to end all Markdown lists and quotes, if inside any, and
2473 * set `quoteMacroLevel` to `0`.
2474 * Params:
2475 * buf = an OutBuffer containing the DDoc
2476 * i = the index within `buf` of the character after the list and/or
2477 * quote text. Is adjusted when this function ends if any lists
2478 * and/or quotes were ended.
2479 * nestedLists = a set of nested lists. Upon return it will be empty.
2480 * quoteLevel = the current quote level. Is set to `0` when this function ends.
2481 * quoteMacroLevel = the macro level that the quote was started at. Is set to
2482 * `0` when this function ends.
2483 * Returns: the amount that `i` was moved
2484 */
endAllListsAndQuotes(ref OutBuffer buf,ref size_t i,ref MarkdownList[]nestedLists,ref int quoteLevel,out int quoteMacroLevel)2485 private size_t endAllListsAndQuotes(ref OutBuffer buf, ref size_t i, ref MarkdownList[] nestedLists, ref int quoteLevel, out int quoteMacroLevel)
2486 {
2487 quoteMacroLevel = 0;
2488 const i0 = i;
2489 i += MarkdownList.endAllNestedLists(buf, i, nestedLists);
2490 i += endAllMarkdownQuotes(buf, i, quoteLevel);
2491 return i - i0;
2492 }
2493
2494 /****************************************************
2495 * Replace Markdown emphasis with the appropriate macro,
2496 * e.g. `*very* **nice**` becomes `$(EM very) $(STRONG nice)`.
2497 * Params:
2498 * buf = an OutBuffer containing the DDoc
2499 * loc = the current location within the file
2500 * inlineDelimiters = the collection of delimiters found within a paragraph. When this function returns its length will be reduced to `downToLevel`.
2501 * downToLevel = the length within `inlineDelimiters`` to reduce emphasis to
2502 * Returns: the number of characters added to the buffer by the replacements
2503 */
2504 private size_t replaceMarkdownEmphasis(ref OutBuffer buf, const ref Loc loc, ref MarkdownDelimiter[] inlineDelimiters, int downToLevel = 0)
2505 {
2506 if (!global.params.markdown)
2507 return 0;
2508
replaceEmphasisPair(ref MarkdownDelimiter start,ref MarkdownDelimiter end)2509 size_t replaceEmphasisPair(ref MarkdownDelimiter start, ref MarkdownDelimiter end)
2510 {
2511 immutable count = start.count == 1 || end.count == 1 ? 1 : 2;
2512
2513 size_t iStart = start.iStart;
2514 size_t iEnd = end.iStart;
2515 end.count -= count;
2516 start.count -= count;
2517 iStart += start.count;
2518
2519 if (!start.count)
2520 start.type = 0;
2521 if (!end.count)
2522 end.type = 0;
2523
2524 if (global.params.vmarkdown)
2525 {
2526 const s = buf[][iStart + count..iEnd];
2527 message(loc, "Ddoc: emphasized text '%.*s'", cast(int)s.length, s.ptr);
2528 }
2529
2530 buf.remove(iStart, count);
2531 iEnd -= count;
2532 buf.remove(iEnd, count);
2533
2534 string macroName = count >= 2 ? "$(STRONG " : "$(EM ";
2535 buf.insert(iEnd, ")");
2536 buf.insert(iStart, macroName);
2537
2538 const delta = 1 + macroName.length - (count + count);
2539 end.iStart += count;
2540 return delta;
2541 }
2542
2543 size_t delta = 0;
2544 int start = (cast(int) inlineDelimiters.length) - 1;
2545 while (start >= downToLevel)
2546 {
2547 // find start emphasis
2548 while (start >= downToLevel &&
2549 (inlineDelimiters[start].type != '*' || !inlineDelimiters[start].leftFlanking))
2550 --start;
2551 if (start < downToLevel)
2552 break;
2553
2554 // find the nearest end emphasis
2555 int end = start + 1;
2556 while (end < inlineDelimiters.length &&
2557 (inlineDelimiters[end].type != inlineDelimiters[start].type ||
2558 inlineDelimiters[end].macroLevel != inlineDelimiters[start].macroLevel ||
2559 !inlineDelimiters[end].rightFlanking))
2560 ++end;
2561 if (end == inlineDelimiters.length)
2562 {
2563 // the start emphasis has no matching end; if it isn't an end itself then kill it
2564 if (!inlineDelimiters[start].rightFlanking)
2565 inlineDelimiters[start].type = 0;
2566 --start;
2567 continue;
2568 }
2569
2570 // multiple-of-3 rule
2571 if (((inlineDelimiters[start].leftFlanking && inlineDelimiters[start].rightFlanking) ||
2572 (inlineDelimiters[end].leftFlanking && inlineDelimiters[end].rightFlanking)) &&
2573 (inlineDelimiters[start].count + inlineDelimiters[end].count) % 3 == 0)
2574 {
2575 --start;
2576 continue;
2577 }
2578
2579 immutable delta0 = replaceEmphasisPair(inlineDelimiters[start], inlineDelimiters[end]);
2580
2581 for (; end < inlineDelimiters.length; ++end)
2582 inlineDelimiters[end].iStart += delta0;
2583 delta += delta0;
2584 }
2585
2586 inlineDelimiters.length = downToLevel;
2587 return delta;
2588 }
2589
2590 /****************************************************
2591 */
isIdentifier(Dsymbols * a,const (char)* p,size_t len)2592 private bool isIdentifier(Dsymbols* a, const(char)* p, size_t len)
2593 {
2594 foreach (member; *a)
2595 {
2596 if (auto imp = member.isImport())
2597 {
2598 // For example: `public import str = core.stdc.string;`
2599 // This checks if `p` is equal to `str`
2600 if (imp.aliasId)
2601 {
2602 if (p[0 .. len] == imp.aliasId.toString())
2603 return true;
2604 }
2605 else
2606 {
2607 // The general case: `public import core.stdc.string;`
2608
2609 // fully qualify imports so `core.stdc.string` doesn't appear as `core`
2610 string fullyQualifiedImport;
2611 foreach (const pid; imp.packages)
2612 {
2613 fullyQualifiedImport ~= pid.toString() ~ ".";
2614 }
2615 fullyQualifiedImport ~= imp.id.toString();
2616
2617 // Check if `p` == `core.stdc.string`
2618 if (p[0 .. len] == fullyQualifiedImport)
2619 return true;
2620 }
2621 }
2622 else if (member.ident)
2623 {
2624 if (p[0 .. len] == member.ident.toString())
2625 return true;
2626 }
2627
2628 }
2629 return false;
2630 }
2631
2632 /****************************************************
2633 */
isKeyword(const (char)* p,size_t len)2634 private bool isKeyword(const(char)* p, size_t len)
2635 {
2636 immutable string[3] table = ["true", "false", "null"];
2637 foreach (s; table)
2638 {
2639 if (p[0 .. len] == s)
2640 return true;
2641 }
2642 return false;
2643 }
2644
2645 /****************************************************
2646 */
isTypeFunction(Dsymbol s)2647 private TypeFunction isTypeFunction(Dsymbol s)
2648 {
2649 FuncDeclaration f = s.isFuncDeclaration();
2650 /* f.type may be NULL for template members.
2651 */
2652 if (f && f.type)
2653 {
2654 Type t = f.originalType ? f.originalType : f.type;
2655 if (t.ty == Tfunction)
2656 return cast(TypeFunction)t;
2657 }
2658 return null;
2659 }
2660
2661 /****************************************************
2662 */
isFunctionParameter(Dsymbol s,const (char)* p,size_t len)2663 private Parameter isFunctionParameter(Dsymbol s, const(char)* p, size_t len)
2664 {
2665 TypeFunction tf = isTypeFunction(s);
2666 if (tf && tf.parameterList.parameters)
2667 {
2668 foreach (fparam; *tf.parameterList.parameters)
2669 {
2670 if (fparam.ident && p[0 .. len] == fparam.ident.toString())
2671 {
2672 return fparam;
2673 }
2674 }
2675 }
2676 return null;
2677 }
2678
2679 /****************************************************
2680 */
isFunctionParameter(Dsymbols * a,const (char)* p,size_t len)2681 private Parameter isFunctionParameter(Dsymbols* a, const(char)* p, size_t len)
2682 {
2683 for (size_t i = 0; i < a.dim; i++)
2684 {
2685 Parameter fparam = isFunctionParameter((*a)[i], p, len);
2686 if (fparam)
2687 {
2688 return fparam;
2689 }
2690 }
2691 return null;
2692 }
2693
2694 /****************************************************
2695 */
isEponymousFunctionParameter(Dsymbols * a,const (char)* p,size_t len)2696 private Parameter isEponymousFunctionParameter(Dsymbols *a, const(char) *p, size_t len)
2697 {
2698 for (size_t i = 0; i < a.dim; i++)
2699 {
2700 TemplateDeclaration td = (*a)[i].isTemplateDeclaration();
2701 if (td && td.onemember)
2702 {
2703 /* Case 1: we refer to a template declaration inside the template
2704
2705 /// ...ddoc...
2706 template case1(T) {
2707 void case1(R)() {}
2708 }
2709 */
2710 td = td.onemember.isTemplateDeclaration();
2711 }
2712 if (!td)
2713 {
2714 /* Case 2: we're an alias to a template declaration
2715
2716 /// ...ddoc...
2717 alias case2 = case1!int;
2718 */
2719 AliasDeclaration ad = (*a)[i].isAliasDeclaration();
2720 if (ad && ad.aliassym)
2721 {
2722 td = ad.aliassym.isTemplateDeclaration();
2723 }
2724 }
2725 while (td)
2726 {
2727 Dsymbol sym = getEponymousMember(td);
2728 if (sym)
2729 {
2730 Parameter fparam = isFunctionParameter(sym, p, len);
2731 if (fparam)
2732 {
2733 return fparam;
2734 }
2735 }
2736 td = td.overnext;
2737 }
2738 }
2739 return null;
2740 }
2741
2742 /****************************************************
2743 */
isTemplateParameter(Dsymbols * a,const (char)* p,size_t len)2744 private TemplateParameter isTemplateParameter(Dsymbols* a, const(char)* p, size_t len)
2745 {
2746 for (size_t i = 0; i < a.dim; i++)
2747 {
2748 TemplateDeclaration td = (*a)[i].isTemplateDeclaration();
2749 // Check for the parent, if the current symbol is not a template declaration.
2750 if (!td)
2751 td = getEponymousParent((*a)[i]);
2752 if (td && td.origParameters)
2753 {
2754 foreach (tp; *td.origParameters)
2755 {
2756 if (tp.ident && p[0 .. len] == tp.ident.toString())
2757 {
2758 return tp;
2759 }
2760 }
2761 }
2762 }
2763 return null;
2764 }
2765
2766 /****************************************************
2767 * Return true if str is a reserved symbol name
2768 * that starts with a double underscore.
2769 */
isReservedName(const (char)[]str)2770 private bool isReservedName(const(char)[] str)
2771 {
2772 immutable string[] table =
2773 [
2774 "__ctor",
2775 "__dtor",
2776 "__postblit",
2777 "__invariant",
2778 "__unitTest",
2779 "__require",
2780 "__ensure",
2781 "__dollar",
2782 "__ctfe",
2783 "__withSym",
2784 "__result",
2785 "__returnLabel",
2786 "__vptr",
2787 "__monitor",
2788 "__gate",
2789 "__xopEquals",
2790 "__xopCmp",
2791 "__LINE__",
2792 "__FILE__",
2793 "__MODULE__",
2794 "__FUNCTION__",
2795 "__PRETTY_FUNCTION__",
2796 "__DATE__",
2797 "__TIME__",
2798 "__TIMESTAMP__",
2799 "__VENDOR__",
2800 "__VERSION__",
2801 "__EOF__",
2802 "__CXXLIB__",
2803 "__LOCAL_SIZE",
2804 "__entrypoint",
2805 ];
2806 foreach (s; table)
2807 {
2808 if (str == s)
2809 return true;
2810 }
2811 return false;
2812 }
2813
2814 /****************************************************
2815 * A delimiter for Markdown inline content like emphasis and links.
2816 */
2817 private struct MarkdownDelimiter
2818 {
2819 size_t iStart; /// the index where this delimiter starts
2820 int count; /// the length of this delimeter's start sequence
2821 int macroLevel; /// the count of nested DDoc macros when the delimiter is started
2822 bool leftFlanking; /// whether the delimiter is left-flanking, as defined by the CommonMark spec
2823 bool rightFlanking; /// whether the delimiter is right-flanking, as defined by the CommonMark spec
2824 bool atParagraphStart; /// whether the delimiter is at the start of a paragraph
2825 char type; /// the type of delimiter, defined by its starting character
2826
2827 /// whether this describes a valid delimiter
isValidMarkdownDelimiter2828 @property bool isValid() const { return count != 0; }
2829
2830 /// flag this delimiter as invalid
invalidateMarkdownDelimiter2831 void invalidate() { count = 0; }
2832 }
2833
2834 /****************************************************
2835 * Info about a Markdown list.
2836 */
2837 private struct MarkdownList
2838 {
2839 string orderedStart; /// an optional start number--if present then the list starts at this number
2840 size_t iStart; /// the index where the list item starts
2841 size_t iContentStart; /// the index where the content starts after the list delimiter
2842 int delimiterIndent; /// the level of indent the list delimiter starts at
2843 int contentIndent; /// the level of indent the content starts at
2844 int macroLevel; /// the count of nested DDoc macros when the list is started
2845 char type; /// the type of list, defined by its starting character
2846
2847 /// whether this describes a valid list
isValid()2848 @property bool isValid() const { return type != type.init; }
2849
2850 /****************************************************
2851 * Try to parse a list item, returning whether successful.
2852 * Params:
2853 * buf = an OutBuffer containing the DDoc
2854 * iLineStart = the index within `buf` of the first character of the line
2855 * i = the index within `buf` of the potential list item
2856 * Returns: the parsed list item. Its `isValid` property describes whether parsing succeeded.
2857 */
parseItem(ref OutBuffer buf,size_t iLineStart,size_t i)2858 static MarkdownList parseItem(ref OutBuffer buf, size_t iLineStart, size_t i)
2859 {
2860 if (!global.params.markdown)
2861 return MarkdownList();
2862
2863 if (buf[i] == '+' || buf[i] == '-' || buf[i] == '*')
2864 return parseUnorderedListItem(buf, iLineStart, i);
2865 else
2866 return parseOrderedListItem(buf, iLineStart, i);
2867 }
2868
2869 /****************************************************
2870 * Return whether the context is at a list item of the same type as this list.
2871 * Params:
2872 * buf = an OutBuffer containing the DDoc
2873 * iLineStart = the index within `buf` of the first character of the line
2874 * i = the index within `buf` of the list item
2875 * Returns: whether `i` is at a list item of the same type as this list
2876 */
isAtItemInThisList(ref OutBuffer buf,size_t iLineStart,size_t i)2877 private bool isAtItemInThisList(ref OutBuffer buf, size_t iLineStart, size_t i)
2878 {
2879 MarkdownList item = (type == '.' || type == ')') ?
2880 parseOrderedListItem(buf, iLineStart, i) :
2881 parseUnorderedListItem(buf, iLineStart, i);
2882 if (item.type == type)
2883 return item.delimiterIndent < contentIndent && item.contentIndent > delimiterIndent;
2884 return false;
2885 }
2886
2887 /****************************************************
2888 * Start a Markdown list item by creating/deleting nested lists and starting the item.
2889 * Params:
2890 * buf = an OutBuffer containing the DDoc
2891 * iLineStart = the index within `buf` of the first character of the line. If this function succeeds it will be adjuested to equal `i`.
2892 * i = the index within `buf` of the list item. If this function succeeds `i` will be adjusted to fit the inserted macro.
2893 * iPrecedingBlankLine = the index within `buf` of the preceeding blank line. If non-zero and a new list was started, the preceeding blank line is removed and this value is set to `0`.
2894 * nestedLists = a set of nested lists. If this function succeeds it may contain a new nested list.
2895 * loc = the location of the Ddoc within the file
2896 * Returns: `true` if a list was created
2897 */
startItem(ref OutBuffer buf,ref size_t iLineStart,ref size_t i,ref size_t iPrecedingBlankLine,ref MarkdownList[]nestedLists,const ref Loc loc)2898 bool startItem(ref OutBuffer buf, ref size_t iLineStart, ref size_t i, ref size_t iPrecedingBlankLine, ref MarkdownList[] nestedLists, const ref Loc loc)
2899 {
2900 buf.remove(iStart, iContentStart - iStart);
2901
2902 if (!nestedLists.length ||
2903 delimiterIndent >= nestedLists[$-1].contentIndent ||
2904 buf[iLineStart - 4..iLineStart] == "$(LI")
2905 {
2906 // start a list macro
2907 nestedLists ~= this;
2908 if (type == '.')
2909 {
2910 if (orderedStart.length)
2911 {
2912 iStart = buf.insert(iStart, "$(OL_START ");
2913 iStart = buf.insert(iStart, orderedStart);
2914 iStart = buf.insert(iStart, ",\n");
2915 }
2916 else
2917 iStart = buf.insert(iStart, "$(OL\n");
2918 }
2919 else
2920 iStart = buf.insert(iStart, "$(UL\n");
2921
2922 removeBlankLineMacro(buf, iPrecedingBlankLine, iStart);
2923 }
2924 else if (nestedLists.length)
2925 {
2926 nestedLists[$-1].delimiterIndent = delimiterIndent;
2927 nestedLists[$-1].contentIndent = contentIndent;
2928 }
2929
2930 iStart = buf.insert(iStart, "$(LI\n");
2931 i = iStart - 1;
2932 iLineStart = i;
2933
2934 if (global.params.vmarkdown)
2935 {
2936 size_t iEnd = iStart;
2937 while (iEnd < buf.length && buf[iEnd] != '\r' && buf[iEnd] != '\n')
2938 ++iEnd;
2939 const s = buf[][iStart..iEnd];
2940 message(loc, "Ddoc: starting list item '%.*s'", cast(int)s.length, s.ptr);
2941 }
2942
2943 return true;
2944 }
2945
2946 /****************************************************
2947 * End all nested Markdown lists.
2948 * Params:
2949 * buf = an OutBuffer containing the DDoc
2950 * i = the index within `buf` to end lists at.
2951 * nestedLists = a set of nested lists. Upon return it will be empty.
2952 * Returns: the amount that `i` changed
2953 */
endAllNestedLists(ref OutBuffer buf,size_t i,ref MarkdownList[]nestedLists)2954 static size_t endAllNestedLists(ref OutBuffer buf, size_t i, ref MarkdownList[] nestedLists)
2955 {
2956 const iStart = i;
2957 for (; nestedLists.length; --nestedLists.length)
2958 i = buf.insert(i, ")\n)");
2959 return i - iStart;
2960 }
2961
2962 /****************************************************
2963 * Look for a sibling list item or the end of nested list(s).
2964 * Params:
2965 * buf = an OutBuffer containing the DDoc
2966 * i = the index within `buf` to end lists at. If there was a sibling or ending lists `i` will be adjusted to fit the macro endings.
2967 * iParagraphStart = the index within `buf` to start the next paragraph at at. May be adjusted upon return.
2968 * nestedLists = a set of nested lists. Some nested lists may have been removed from it upon return.
2969 */
handleSiblingOrEndingList(ref OutBuffer buf,ref size_t i,ref size_t iParagraphStart,ref MarkdownList[]nestedLists)2970 static void handleSiblingOrEndingList(ref OutBuffer buf, ref size_t i, ref size_t iParagraphStart, ref MarkdownList[] nestedLists)
2971 {
2972 size_t iAfterSpaces = skipChars(buf, i + 1, " \t");
2973
2974 if (nestedLists[$-1].isAtItemInThisList(buf, i + 1, iAfterSpaces))
2975 {
2976 // end a sibling list item
2977 i = buf.insert(i, ")");
2978 iParagraphStart = skipChars(buf, i, " \t\r\n");
2979 }
2980 else if (iAfterSpaces >= buf.length || (buf[iAfterSpaces] != '\r' && buf[iAfterSpaces] != '\n'))
2981 {
2982 // end nested lists that are indented more than this content
2983 const indent = getMarkdownIndent(buf, i + 1, iAfterSpaces);
2984 while (nestedLists.length && nestedLists[$-1].contentIndent > indent)
2985 {
2986 i = buf.insert(i, ")\n)");
2987 --nestedLists.length;
2988 iParagraphStart = skipChars(buf, i, " \t\r\n");
2989
2990 if (nestedLists.length && nestedLists[$-1].isAtItemInThisList(buf, i + 1, iParagraphStart))
2991 {
2992 i = buf.insert(i, ")");
2993 ++iParagraphStart;
2994 break;
2995 }
2996 }
2997 }
2998 }
2999
3000 /****************************************************
3001 * Parse an unordered list item at the current position
3002 * Params:
3003 * buf = an OutBuffer containing the DDoc
3004 * iLineStart = the index within `buf` of the first character of the line
3005 * i = the index within `buf` of the list item
3006 * Returns: the parsed list item, or a list item with type `.init` if no list item is available
3007 */
parseUnorderedListItem(ref OutBuffer buf,size_t iLineStart,size_t i)3008 private static MarkdownList parseUnorderedListItem(ref OutBuffer buf, size_t iLineStart, size_t i)
3009 {
3010 if (i+1 < buf.length &&
3011 (buf[i] == '-' ||
3012 buf[i] == '*' ||
3013 buf[i] == '+') &&
3014 (buf[i+1] == ' ' ||
3015 buf[i+1] == '\t' ||
3016 buf[i+1] == '\r' ||
3017 buf[i+1] == '\n'))
3018 {
3019 const iContentStart = skipChars(buf, i + 1, " \t");
3020 const delimiterIndent = getMarkdownIndent(buf, iLineStart, i);
3021 const contentIndent = getMarkdownIndent(buf, iLineStart, iContentStart);
3022 auto list = MarkdownList(null, iLineStart, iContentStart, delimiterIndent, contentIndent, 0, buf[i]);
3023 return list;
3024 }
3025 return MarkdownList();
3026 }
3027
3028 /****************************************************
3029 * Parse an ordered list item at the current position
3030 * Params:
3031 * buf = an OutBuffer containing the DDoc
3032 * iLineStart = the index within `buf` of the first character of the line
3033 * i = the index within `buf` of the list item
3034 * Returns: the parsed list item, or a list item with type `.init` if no list item is available
3035 */
parseOrderedListItem(ref OutBuffer buf,size_t iLineStart,size_t i)3036 private static MarkdownList parseOrderedListItem(ref OutBuffer buf, size_t iLineStart, size_t i)
3037 {
3038 size_t iAfterNumbers = skipChars(buf, i, "0123456789");
3039 if (iAfterNumbers - i > 0 &&
3040 iAfterNumbers - i <= 9 &&
3041 iAfterNumbers + 1 < buf.length &&
3042 buf[iAfterNumbers] == '.' &&
3043 (buf[iAfterNumbers+1] == ' ' ||
3044 buf[iAfterNumbers+1] == '\t' ||
3045 buf[iAfterNumbers+1] == '\r' ||
3046 buf[iAfterNumbers+1] == '\n'))
3047 {
3048 const iContentStart = skipChars(buf, iAfterNumbers + 1, " \t");
3049 const delimiterIndent = getMarkdownIndent(buf, iLineStart, i);
3050 const contentIndent = getMarkdownIndent(buf, iLineStart, iContentStart);
3051 size_t iNumberStart = skipChars(buf, i, "0");
3052 if (iNumberStart == iAfterNumbers)
3053 --iNumberStart;
3054 auto orderedStart = buf[][iNumberStart .. iAfterNumbers];
3055 if (orderedStart == "1")
3056 orderedStart = null;
3057 return MarkdownList(orderedStart.idup, iLineStart, iContentStart, delimiterIndent, contentIndent, 0, buf[iAfterNumbers]);
3058 }
3059 return MarkdownList();
3060 }
3061 }
3062
3063 /****************************************************
3064 * A Markdown link.
3065 */
3066 private struct MarkdownLink
3067 {
3068 string href; /// the link destination
3069 string title; /// an optional title for the link
3070 string label; /// an optional label for the link
3071 Dsymbol symbol; /// an optional symbol to link to
3072
3073 /****************************************************
3074 * Replace a Markdown link or link definition in the form of:
3075 * - Inline link: `[foo](url/ 'optional title')`
3076 * - Reference link: `[foo][bar]`, `[foo][]` or `[foo]`
3077 * - Link reference definition: `[bar]: url/ 'optional title'`
3078 * Params:
3079 * buf = an OutBuffer containing the DDoc
3080 * i = the index within `buf` that points to the `]` character of the potential link.
3081 * If this function succeeds it will be adjusted to fit the inserted link macro.
3082 * loc = the current location within the file
3083 * inlineDelimiters = previously parsed Markdown delimiters, including emphasis and link/image starts
3084 * delimiterIndex = the index within `inlineDelimiters` of the nearest link/image starting delimiter
3085 * linkReferences = previously parsed link references. When this function returns it may contain
3086 * additional previously unparsed references.
3087 * Returns: whether a reference link was found and replaced at `i`
3088 */
replaceLinkMarkdownLink3089 static bool replaceLink(ref OutBuffer buf, ref size_t i, const ref Loc loc, ref MarkdownDelimiter[] inlineDelimiters, int delimiterIndex, ref MarkdownLinkReferences linkReferences)
3090 {
3091 const delimiter = inlineDelimiters[delimiterIndex];
3092 MarkdownLink link;
3093
3094 size_t iEnd = link.parseReferenceDefinition(buf, i, delimiter);
3095 if (iEnd > i)
3096 {
3097 i = delimiter.iStart;
3098 link.storeAndReplaceDefinition(buf, i, iEnd, linkReferences, loc);
3099 inlineDelimiters.length = delimiterIndex;
3100 return true;
3101 }
3102
3103 iEnd = link.parseInlineLink(buf, i);
3104 if (iEnd == i)
3105 {
3106 iEnd = link.parseReferenceLink(buf, i, delimiter);
3107 if (iEnd > i)
3108 {
3109 const label = link.label;
3110 link = linkReferences.lookupReference(label, buf, i, loc);
3111 // check rightFlanking to avoid replacing things like int[string]
3112 if (!link.href.length && !delimiter.rightFlanking)
3113 link = linkReferences.lookupSymbol(label);
3114 if (!link.href.length)
3115 return false;
3116 }
3117 }
3118
3119 if (iEnd == i)
3120 return false;
3121
3122 immutable delta = replaceMarkdownEmphasis(buf, loc, inlineDelimiters, delimiterIndex);
3123 iEnd += delta;
3124 i += delta;
3125
3126 if (global.params.vmarkdown)
3127 {
3128 const s = buf[][delimiter.iStart..iEnd];
3129 message(loc, "Ddoc: linking '%.*s' to '%.*s'", cast(int)s.length, s.ptr, cast(int)link.href.length, link.href.ptr);
3130 }
3131
3132 link.replaceLink(buf, i, iEnd, delimiter);
3133 return true;
3134 }
3135
3136 /****************************************************
3137 * Replace a Markdown link definition in the form of `[bar]: url/ 'optional title'`
3138 * Params:
3139 * buf = an OutBuffer containing the DDoc
3140 * i = the index within `buf` that points to the `]` character of the potential link.
3141 * If this function succeeds it will be adjusted to fit the inserted link macro.
3142 * inlineDelimiters = previously parsed Markdown delimiters, including emphasis and link/image starts
3143 * delimiterIndex = the index within `inlineDelimiters` of the nearest link/image starting delimiter
3144 * linkReferences = previously parsed link references. When this function returns it may contain
3145 * additional previously unparsed references.
3146 * loc = the current location in the file
3147 * Returns: whether a reference link was found and replaced at `i`
3148 */
replaceReferenceDefinitionMarkdownLink3149 static bool replaceReferenceDefinition(ref OutBuffer buf, ref size_t i, ref MarkdownDelimiter[] inlineDelimiters, int delimiterIndex, ref MarkdownLinkReferences linkReferences, const ref Loc loc)
3150 {
3151 const delimiter = inlineDelimiters[delimiterIndex];
3152 MarkdownLink link;
3153 size_t iEnd = link.parseReferenceDefinition(buf, i, delimiter);
3154 if (iEnd == i)
3155 return false;
3156
3157 i = delimiter.iStart;
3158 link.storeAndReplaceDefinition(buf, i, iEnd, linkReferences, loc);
3159 inlineDelimiters.length = delimiterIndex;
3160 return true;
3161 }
3162
3163 /****************************************************
3164 * Parse a Markdown inline link in the form of `[foo](url/ 'optional title')`
3165 * Params:
3166 * buf = an OutBuffer containing the DDoc
3167 * i = the index within `buf` that points to the `]` character of the inline link.
3168 * Returns: the index at the end of parsing the link, or `i` if parsing failed.
3169 */
parseInlineLinkMarkdownLink3170 private size_t parseInlineLink(ref OutBuffer buf, size_t i)
3171 {
3172 size_t iEnd = i + 1;
3173 if (iEnd >= buf.length || buf[iEnd] != '(')
3174 return i;
3175 ++iEnd;
3176
3177 if (!parseHref(buf, iEnd))
3178 return i;
3179
3180 iEnd = skipChars(buf, iEnd, " \t\r\n");
3181 if (buf[iEnd] != ')')
3182 {
3183 if (parseTitle(buf, iEnd))
3184 iEnd = skipChars(buf, iEnd, " \t\r\n");
3185 }
3186
3187 if (buf[iEnd] != ')')
3188 return i;
3189
3190 return iEnd + 1;
3191 }
3192
3193 /****************************************************
3194 * Parse a Markdown reference link in the form of `[foo][bar]`, `[foo][]` or `[foo]`
3195 * Params:
3196 * buf = an OutBuffer containing the DDoc
3197 * i = the index within `buf` that points to the `]` character of the inline link.
3198 * delimiter = the delimiter that starts this link
3199 * Returns: the index at the end of parsing the link, or `i` if parsing failed.
3200 */
parseReferenceLinkMarkdownLink3201 private size_t parseReferenceLink(ref OutBuffer buf, size_t i, MarkdownDelimiter delimiter)
3202 {
3203 size_t iStart = i + 1;
3204 size_t iEnd = iStart;
3205 if (iEnd >= buf.length || buf[iEnd] != '[' || (iEnd+1 < buf.length && buf[iEnd+1] == ']'))
3206 {
3207 // collapsed reference [foo][] or shortcut reference [foo]
3208 iStart = delimiter.iStart + delimiter.count - 1;
3209 if (buf[iEnd] == '[')
3210 iEnd += 2;
3211 }
3212
3213 parseLabel(buf, iStart);
3214 if (!label.length)
3215 return i;
3216
3217 if (iEnd < iStart)
3218 iEnd = iStart;
3219 return iEnd;
3220 }
3221
3222 /****************************************************
3223 * Parse a Markdown reference definition in the form of `[bar]: url/ 'optional title'`
3224 * Params:
3225 * buf = an OutBuffer containing the DDoc
3226 * i = the index within `buf` that points to the `]` character of the inline link.
3227 * delimiter = the delimiter that starts this link
3228 * Returns: the index at the end of parsing the link, or `i` if parsing failed.
3229 */
parseReferenceDefinitionMarkdownLink3230 private size_t parseReferenceDefinition(ref OutBuffer buf, size_t i, MarkdownDelimiter delimiter)
3231 {
3232 if (!delimiter.atParagraphStart || delimiter.type != '[' ||
3233 i+1 >= buf.length || buf[i+1] != ':')
3234 return i;
3235
3236 size_t iEnd = delimiter.iStart;
3237 parseLabel(buf, iEnd);
3238 if (label.length == 0 || iEnd != i + 1)
3239 return i;
3240
3241 ++iEnd;
3242 iEnd = skipChars(buf, iEnd, " \t");
3243 skipOneNewline(buf, iEnd);
3244
3245 if (!parseHref(buf, iEnd) || href.length == 0)
3246 return i;
3247
3248 iEnd = skipChars(buf, iEnd, " \t");
3249 const requireNewline = !skipOneNewline(buf, iEnd);
3250 const iBeforeTitle = iEnd;
3251
3252 if (parseTitle(buf, iEnd))
3253 {
3254 iEnd = skipChars(buf, iEnd, " \t");
3255 if (iEnd < buf.length && buf[iEnd] != '\r' && buf[iEnd] != '\n')
3256 {
3257 // the title must end with a newline
3258 title.length = 0;
3259 iEnd = iBeforeTitle;
3260 }
3261 }
3262
3263 iEnd = skipChars(buf, iEnd, " \t");
3264 if (requireNewline && iEnd < buf.length-1 && buf[iEnd] != '\r' && buf[iEnd] != '\n')
3265 return i;
3266
3267 return iEnd;
3268 }
3269
3270 /****************************************************
3271 * Parse and normalize a Markdown reference label
3272 * Params:
3273 * buf = an OutBuffer containing the DDoc
3274 * i = the index within `buf` that points to the `[` character at the start of the label.
3275 * If this function returns a non-empty label then `i` will point just after the ']' at the end of the label.
3276 * Returns: the parsed and normalized label, possibly empty
3277 */
parseLabelMarkdownLink3278 private bool parseLabel(ref OutBuffer buf, ref size_t i)
3279 {
3280 if (buf[i] != '[')
3281 return false;
3282
3283 const slice = buf[];
3284 size_t j = i + 1;
3285
3286 // Some labels have already been en-symboled; handle that
3287 const inSymbol = j+15 < slice.length && slice[j..j+15] == "$(DDOC_PSYMBOL ";
3288 if (inSymbol)
3289 j += 15;
3290
3291 for (; j < slice.length; ++j)
3292 {
3293 const c = slice[j];
3294 switch (c)
3295 {
3296 case ' ':
3297 case '\t':
3298 case '\r':
3299 case '\n':
3300 if (label.length && label[$-1] != ' ')
3301 label ~= ' ';
3302 break;
3303 case ')':
3304 if (inSymbol && j+1 < slice.length && slice[j+1] == ']')
3305 {
3306 ++j;
3307 goto case ']';
3308 }
3309 goto default;
3310 case '[':
3311 if (slice[j-1] != '\\')
3312 {
3313 label.length = 0;
3314 return false;
3315 }
3316 break;
3317 case ']':
3318 if (label.length && label[$-1] == ' ')
3319 --label.length;
3320 if (label.length)
3321 {
3322 i = j + 1;
3323 return true;
3324 }
3325 return false;
3326 default:
3327 label ~= c;
3328 break;
3329 }
3330 }
3331 label.length = 0;
3332 return false;
3333 }
3334
3335 /****************************************************
3336 * Parse and store a Markdown link URL, optionally enclosed in `<>` brackets
3337 * Params:
3338 * buf = an OutBuffer containing the DDoc
3339 * i = the index within `buf` that points to the first character of the URL.
3340 * If this function succeeds `i` will point just after the the end of the URL.
3341 * Returns: whether a URL was found and parsed
3342 */
parseHrefMarkdownLink3343 private bool parseHref(ref OutBuffer buf, ref size_t i)
3344 {
3345 size_t j = skipChars(buf, i, " \t");
3346
3347 size_t iHrefStart = j;
3348 size_t parenDepth = 1;
3349 bool inPointy = false;
3350 const slice = buf[];
3351 for (; j < slice.length; j++)
3352 {
3353 switch (slice[j])
3354 {
3355 case '<':
3356 if (!inPointy && j == iHrefStart)
3357 {
3358 inPointy = true;
3359 ++iHrefStart;
3360 }
3361 break;
3362 case '>':
3363 if (inPointy && slice[j-1] != '\\')
3364 goto LReturnHref;
3365 break;
3366 case '(':
3367 if (!inPointy && slice[j-1] != '\\')
3368 ++parenDepth;
3369 break;
3370 case ')':
3371 if (!inPointy && slice[j-1] != '\\')
3372 {
3373 --parenDepth;
3374 if (!parenDepth)
3375 goto LReturnHref;
3376 }
3377 break;
3378 case ' ':
3379 case '\t':
3380 case '\r':
3381 case '\n':
3382 if (inPointy)
3383 {
3384 // invalid link
3385 return false;
3386 }
3387 goto LReturnHref;
3388 default:
3389 break;
3390 }
3391 }
3392 if (inPointy)
3393 return false;
3394 LReturnHref:
3395 auto href = slice[iHrefStart .. j].dup;
3396 this.href = cast(string) percentEncode(removeEscapeBackslashes(href)).replaceChar(',', "$(COMMA)");
3397 i = j;
3398 if (inPointy)
3399 ++i;
3400 return true;
3401 }
3402
3403 /****************************************************
3404 * Parse and store a Markdown link title, enclosed in parentheses or `'` or `"` quotes
3405 * Params:
3406 * buf = an OutBuffer containing the DDoc
3407 * i = the index within `buf` that points to the first character of the title.
3408 * If this function succeeds `i` will point just after the the end of the title.
3409 * Returns: whether a title was found and parsed
3410 */
parseTitleMarkdownLink3411 private bool parseTitle(ref OutBuffer buf, ref size_t i)
3412 {
3413 size_t j = skipChars(buf, i, " \t");
3414 if (j >= buf.length)
3415 return false;
3416
3417 char type = buf[j];
3418 if (type != '"' && type != '\'' && type != '(')
3419 return false;
3420 if (type == '(')
3421 type = ')';
3422
3423 const iTitleStart = j + 1;
3424 size_t iNewline = 0;
3425 const slice = buf[];
3426 for (j = iTitleStart; j < slice.length; j++)
3427 {
3428 const c = slice[j];
3429 switch (c)
3430 {
3431 case ')':
3432 case '"':
3433 case '\'':
3434 if (type == c && slice[j-1] != '\\')
3435 goto LEndTitle;
3436 iNewline = 0;
3437 break;
3438 case ' ':
3439 case '\t':
3440 case '\r':
3441 break;
3442 case '\n':
3443 if (iNewline)
3444 {
3445 // no blank lines in titles
3446 return false;
3447 }
3448 iNewline = j;
3449 break;
3450 default:
3451 iNewline = 0;
3452 break;
3453 }
3454 }
3455 return false;
3456 LEndTitle:
3457 auto title = slice[iTitleStart .. j].dup;
3458 this.title = cast(string) removeEscapeBackslashes(title).
3459 replaceChar(',', "$(COMMA)").
3460 replaceChar('"', "$(QUOTE)");
3461 i = j + 1;
3462 return true;
3463 }
3464
3465 /****************************************************
3466 * Replace a Markdown link or image with the appropriate macro
3467 * Params:
3468 * buf = an OutBuffer containing the DDoc
3469 * i = the index within `buf` that points to the `]` character of the inline link.
3470 * When this function returns it will be adjusted to the end of the inserted macro.
3471 * iLinkEnd = the index within `buf` that points just after the last character of the link
3472 * delimiter = the Markdown delimiter that started the link or image
3473 */
replaceLinkMarkdownLink3474 private void replaceLink(ref OutBuffer buf, ref size_t i, size_t iLinkEnd, MarkdownDelimiter delimiter)
3475 {
3476 size_t iAfterLink = i - delimiter.count;
3477 string macroName;
3478 if (symbol)
3479 {
3480 macroName = "$(SYMBOL_LINK ";
3481 }
3482 else if (title.length)
3483 {
3484 if (delimiter.type == '[')
3485 macroName = "$(LINK_TITLE ";
3486 else
3487 macroName = "$(IMAGE_TITLE ";
3488 }
3489 else
3490 {
3491 if (delimiter.type == '[')
3492 macroName = "$(LINK2 ";
3493 else
3494 macroName = "$(IMAGE ";
3495 }
3496 buf.remove(delimiter.iStart, delimiter.count);
3497 buf.remove(i - delimiter.count, iLinkEnd - i);
3498 iLinkEnd = buf.insert(delimiter.iStart, macroName);
3499 iLinkEnd = buf.insert(iLinkEnd, href);
3500 iLinkEnd = buf.insert(iLinkEnd, ", ");
3501 iAfterLink += macroName.length + href.length + 2;
3502 if (title.length)
3503 {
3504 iLinkEnd = buf.insert(iLinkEnd, title);
3505 iLinkEnd = buf.insert(iLinkEnd, ", ");
3506 iAfterLink += title.length + 2;
3507
3508 // Link macros with titles require escaping commas
3509 for (size_t j = iLinkEnd; j < iAfterLink; ++j)
3510 if (buf[j] == ',')
3511 {
3512 buf.remove(j, 1);
3513 j = buf.insert(j, "$(COMMA)") - 1;
3514 iAfterLink += 7;
3515 }
3516 }
3517 // TODO: if image, remove internal macros, leaving only text
3518 buf.insert(iAfterLink, ")");
3519 i = iAfterLink;
3520 }
3521
3522 /****************************************************
3523 * Store the Markdown link definition and remove it from `buf`
3524 * Params:
3525 * buf = an OutBuffer containing the DDoc
3526 * i = the index within `buf` that points to the `[` character at the start of the link definition.
3527 * When this function returns it will be adjusted to exclude the link definition.
3528 * iEnd = the index within `buf` that points just after the end of the definition
3529 * linkReferences = previously parsed link references. When this function returns it may contain
3530 * an additional reference.
3531 * loc = the current location in the file
3532 */
storeAndReplaceDefinitionMarkdownLink3533 private void storeAndReplaceDefinition(ref OutBuffer buf, ref size_t i, size_t iEnd, ref MarkdownLinkReferences linkReferences, const ref Loc loc)
3534 {
3535 if (global.params.vmarkdown)
3536 message(loc, "Ddoc: found link reference '%.*s' to '%.*s'", cast(int)label.length, label.ptr, cast(int)href.length, href.ptr);
3537
3538 // Remove the definition and trailing whitespace
3539 iEnd = skipChars(buf, iEnd, " \t\r\n");
3540 buf.remove(i, iEnd - i);
3541 i -= 2;
3542
3543 string lowercaseLabel = label.toLowercase();
3544 if (lowercaseLabel !in linkReferences.references)
3545 linkReferences.references[lowercaseLabel] = this;
3546 }
3547
3548 /****************************************************
3549 * Remove Markdown escaping backslashes from the given string
3550 * Params:
3551 * s = the string to remove escaping backslashes from
3552 * Returns: `s` without escaping backslashes in it
3553 */
removeEscapeBackslashesMarkdownLink3554 private static char[] removeEscapeBackslashes(char[] s)
3555 {
3556 if (!s.length)
3557 return s;
3558
3559 // avoid doing anything if there isn't anything to escape
3560 size_t i;
3561 for (i = 0; i < s.length-1; ++i)
3562 if (s[i] == '\\' && ispunct(s[i+1]))
3563 break;
3564 if (i == s.length-1)
3565 return s;
3566
3567 // copy characters backwards, then truncate
3568 size_t j = i + 1;
3569 s[i] = s[j];
3570 for (++i, ++j; j < s.length; ++i, ++j)
3571 {
3572 if (j < s.length-1 && s[j] == '\\' && ispunct(s[j+1]))
3573 ++j;
3574 s[i] = s[j];
3575 }
3576 s.length -= (j - i);
3577 return s;
3578 }
3579
3580 ///
3581 unittest
3582 {
3583 assert(removeEscapeBackslashes("".dup) == "");
3584 assert(removeEscapeBackslashes(`\a`.dup) == `\a`);
3585 assert(removeEscapeBackslashes(`.\`.dup) == `.\`);
3586 assert(removeEscapeBackslashes(`\.\`.dup) == `.\`);
3587 assert(removeEscapeBackslashes(`\.`.dup) == `.`);
3588 assert(removeEscapeBackslashes(`\.\.`.dup) == `..`);
3589 assert(removeEscapeBackslashes(`a\.b\.c`.dup) == `a.b.c`);
3590 }
3591
3592 /****************************************************
3593 * Percent-encode (AKA URL-encode) the given string
3594 * Params:
3595 * s = the string to percent-encode
3596 * Returns: `s` with special characters percent-encoded
3597 */
inoutMarkdownLink3598 private static inout(char)[] percentEncode(inout(char)[] s) pure
3599 {
3600 static bool shouldEncode(char c)
3601 {
3602 return ((c < '0' && c != '!' && c != '#' && c != '$' && c != '%' && c != '&' && c != '\'' && c != '(' &&
3603 c != ')' && c != '*' && c != '+' && c != ',' && c != '-' && c != '.' && c != '/')
3604 || (c > '9' && c < 'A' && c != ':' && c != ';' && c != '=' && c != '?' && c != '@')
3605 || (c > 'Z' && c < 'a' && c != '[' && c != ']' && c != '_')
3606 || (c > 'z' && c != '~'));
3607 }
3608
3609 for (size_t i = 0; i < s.length; ++i)
3610 {
3611 if (shouldEncode(s[i]))
3612 {
3613 immutable static hexDigits = "0123456789ABCDEF";
3614 immutable encoded1 = hexDigits[s[i] >> 4];
3615 immutable encoded2 = hexDigits[s[i] & 0x0F];
3616 s = s[0..i] ~ '%' ~ encoded1 ~ encoded2 ~ s[i+1..$];
3617 i += 2;
3618 }
3619 }
3620 return s;
3621 }
3622
3623 ///
3624 unittest
3625 {
3626 assert(percentEncode("") == "");
3627 assert(percentEncode("aB12-._~/?") == "aB12-._~/?");
3628 assert(percentEncode("<\n>") == "%3C%0A%3E");
3629 }
3630
3631 /**************************************************
3632 * Skip a single newline at `i`
3633 * Params:
3634 * buf = an OutBuffer containing the DDoc
3635 * i = the index within `buf` to start looking at.
3636 * If this function succeeds `i` will point after the newline.
3637 * Returns: whether a newline was skipped
3638 */
skipOneNewlineMarkdownLink3639 private static bool skipOneNewline(ref OutBuffer buf, ref size_t i) pure
3640 {
3641 if (i < buf.length && buf[i] == '\r')
3642 ++i;
3643 if (i < buf.length && buf[i] == '\n')
3644 {
3645 ++i;
3646 return true;
3647 }
3648 return false;
3649 }
3650 }
3651
3652 /**************************************************
3653 * A set of Markdown link references.
3654 */
3655 private struct MarkdownLinkReferences
3656 {
3657 MarkdownLink[string] references; // link references keyed by normalized label
3658 MarkdownLink[string] symbols; // link symbols keyed by name
3659 Scope* _scope; // the current scope
3660 bool extractedAll; // the index into the buffer of the last-parsed reference
3661
3662 /**************************************************
3663 * Look up a reference by label, searching through the rest of the buffer if needed.
3664 * Symbols in the current scope are searched for if the DDoc doesn't define the reference.
3665 * Params:
3666 * label = the label to find the reference for
3667 * buf = an OutBuffer containing the DDoc
3668 * i = the index within `buf` to start searching for references at
3669 * loc = the current location in the file
3670 * Returns: a link. If the `href` member has a value then the reference is valid.
3671 */
lookupReference(string label,ref OutBuffer buf,size_t i,const ref Loc loc)3672 MarkdownLink lookupReference(string label, ref OutBuffer buf, size_t i, const ref Loc loc)
3673 {
3674 const lowercaseLabel = label.toLowercase();
3675 if (lowercaseLabel !in references)
3676 extractReferences(buf, i, loc);
3677
3678 if (lowercaseLabel in references)
3679 return references[lowercaseLabel];
3680
3681 return MarkdownLink();
3682 }
3683
3684 /**
3685 * Look up the link for the D symbol with the given name.
3686 * If found, the link is cached in the `symbols` member.
3687 * Params:
3688 * name = the name of the symbol
3689 * Returns: the link for the symbol or a link with a `null` href
3690 */
lookupSymbol(string name)3691 MarkdownLink lookupSymbol(string name)
3692 {
3693 if (name in symbols)
3694 return symbols[name];
3695
3696 const ids = split(name, '.');
3697
3698 MarkdownLink link;
3699 auto id = Identifier.lookup(ids[0].ptr, ids[0].length);
3700 if (id)
3701 {
3702 auto loc = Loc();
3703 auto symbol = _scope.search(loc, id, null, IgnoreErrors);
3704 for (size_t i = 1; symbol && i < ids.length; ++i)
3705 {
3706 id = Identifier.lookup(ids[i].ptr, ids[i].length);
3707 symbol = id !is null ? symbol.search(loc, id, IgnoreErrors) : null;
3708 }
3709 if (symbol)
3710 link = MarkdownLink(createHref(symbol), null, name, symbol);
3711 }
3712
3713 symbols[name] = link;
3714 return link;
3715 }
3716
3717 /**************************************************
3718 * Remove and store all link references from the document, in the form of
3719 * `[label]: href "optional title"`
3720 * Params:
3721 * buf = an OutBuffer containing the DDoc
3722 * i = the index within `buf` to start looking at
3723 * loc = the current location in the file
3724 * Returns: whether a reference was extracted
3725 */
extractReferences(ref OutBuffer buf,size_t i,const ref Loc loc)3726 private void extractReferences(ref OutBuffer buf, size_t i, const ref Loc loc)
3727 {
3728 static bool isFollowedBySpace(ref OutBuffer buf, size_t i)
3729 {
3730 return i+1 < buf.length && (buf[i+1] == ' ' || buf[i+1] == '\t');
3731 }
3732
3733 if (extractedAll)
3734 return;
3735
3736 bool leadingBlank = false;
3737 int inCode = false;
3738 bool newParagraph = true;
3739 MarkdownDelimiter[] delimiters;
3740 for (; i < buf.length; ++i)
3741 {
3742 const c = buf[i];
3743 switch (c)
3744 {
3745 case ' ':
3746 case '\t':
3747 break;
3748 case '\n':
3749 if (leadingBlank && !inCode)
3750 newParagraph = true;
3751 leadingBlank = true;
3752 break;
3753 case '\\':
3754 ++i;
3755 break;
3756 case '#':
3757 if (leadingBlank && !inCode)
3758 newParagraph = true;
3759 leadingBlank = false;
3760 break;
3761 case '>':
3762 if (leadingBlank && !inCode)
3763 newParagraph = true;
3764 break;
3765 case '+':
3766 if (leadingBlank && !inCode && isFollowedBySpace(buf, i))
3767 newParagraph = true;
3768 else
3769 leadingBlank = false;
3770 break;
3771 case '0':
3772 ..
3773 case '9':
3774 if (leadingBlank && !inCode)
3775 {
3776 i = skipChars(buf, i, "0123456789");
3777 if (i < buf.length &&
3778 (buf[i] == '.' || buf[i] == ')') &&
3779 isFollowedBySpace(buf, i))
3780 newParagraph = true;
3781 else
3782 leadingBlank = false;
3783 }
3784 break;
3785 case '*':
3786 if (leadingBlank && !inCode)
3787 {
3788 newParagraph = true;
3789 if (!isFollowedBySpace(buf, i))
3790 leadingBlank = false;
3791 }
3792 break;
3793 case '`':
3794 case '~':
3795 if (leadingBlank && i+2 < buf.length && buf[i+1] == c && buf[i+2] == c)
3796 {
3797 inCode = inCode == c ? false : c;
3798 i = skipChars(buf, i, [c]) - 1;
3799 newParagraph = true;
3800 }
3801 leadingBlank = false;
3802 break;
3803 case '-':
3804 if (leadingBlank && !inCode && isFollowedBySpace(buf, i))
3805 goto case '+';
3806 else
3807 goto case '`';
3808 case '[':
3809 if (leadingBlank && !inCode && newParagraph)
3810 delimiters ~= MarkdownDelimiter(i, 1, 0, false, false, true, c);
3811 break;
3812 case ']':
3813 if (delimiters.length && !inCode &&
3814 MarkdownLink.replaceReferenceDefinition(buf, i, delimiters, cast(int) delimiters.length - 1, this, loc))
3815 --i;
3816 break;
3817 default:
3818 if (leadingBlank)
3819 newParagraph = false;
3820 leadingBlank = false;
3821 break;
3822 }
3823 }
3824 extractedAll = true;
3825 }
3826
3827 /**
3828 * Split a string by a delimiter, excluding the delimiter.
3829 * Params:
3830 * s = the string to split
3831 * delimiter = the character to split by
3832 * Returns: the resulting array of strings
3833 */
split(string s,char delimiter)3834 private static string[] split(string s, char delimiter) pure
3835 {
3836 string[] result;
3837 size_t iStart = 0;
3838 foreach (size_t i; 0..s.length)
3839 if (s[i] == delimiter)
3840 {
3841 result ~= s[iStart..i];
3842 iStart = i + 1;
3843 }
3844 result ~= s[iStart..$];
3845 return result;
3846 }
3847
3848 ///
3849 unittest
3850 {
3851 assert(split("", ',') == [""]);
3852 assert(split("ab", ',') == ["ab"]);
3853 assert(split("a,b", ',') == ["a", "b"]);
3854 assert(split("a,,b", ',') == ["a", "", "b"]);
3855 assert(split(",ab", ',') == ["", "ab"]);
3856 assert(split("ab,", ',') == ["ab", ""]);
3857 }
3858
3859 /**
3860 * Create a HREF for the given D symbol.
3861 * The HREF is relative to the current location if possible.
3862 * Params:
3863 * symbol = the symbol to create a HREF for.
3864 * Returns: the resulting href
3865 */
createHref(Dsymbol symbol)3866 private string createHref(Dsymbol symbol)
3867 {
3868 Dsymbol root = symbol;
3869
3870 const(char)[] lref;
3871 while (symbol && symbol.ident && !symbol.isModule())
3872 {
3873 if (lref.length)
3874 lref = '.' ~ lref;
3875 lref = symbol.ident.toString() ~ lref;
3876 symbol = symbol.parent;
3877 }
3878
3879 const(char)[] path;
3880 if (symbol && symbol.ident && symbol.isModule() != _scope._module)
3881 {
3882 do
3883 {
3884 root = symbol;
3885
3886 // If the module has a file name, we're done
3887 if (const m = symbol.isModule())
3888 if (m.docfile)
3889 {
3890 path = m.docfile.toString();
3891 break;
3892 }
3893
3894 if (path.length)
3895 path = '_' ~ path;
3896 path = symbol.ident.toString() ~ path;
3897 symbol = symbol.parent;
3898 } while (symbol && symbol.ident);
3899
3900 if (!symbol && path.length)
3901 path ~= "$(DOC_EXTENSION)";
3902 }
3903
3904 // Attempt an absolute URL if not in the same package
3905 while (root.parent)
3906 root = root.parent;
3907 Dsymbol scopeRoot = _scope._module;
3908 while (scopeRoot.parent)
3909 scopeRoot = scopeRoot.parent;
3910 if (scopeRoot != root)
3911 {
3912 path = "$(DOC_ROOT_" ~ root.ident.toString() ~ ')' ~ path;
3913 lref = '.' ~ lref; // remote URIs like Phobos and Mir use .prefixes
3914 }
3915
3916 return cast(string) (path ~ '#' ~ lref);
3917 }
3918 }
3919
3920 private enum TableColumnAlignment
3921 {
3922 none,
3923 left,
3924 center,
3925 right
3926 }
3927
3928 /****************************************************
3929 * Parse a Markdown table delimiter row in the form of `| -- | :-- | :--: | --: |`
3930 * where the example text has four columns with the following alignments:
3931 * default, left, center, and right. The first and last pipes are optional. If a
3932 * delimiter row is found it will be removed from `buf`.
3933 *
3934 * Params:
3935 * buf = an OutBuffer containing the DDoc
3936 * iStart = the index within `buf` that the delimiter row starts at
3937 * inQuote = whether the table is inside a quote
3938 * columnAlignments = alignments to populate for each column
3939 * Returns: the index of the end of the parsed delimiter, or `0` if not found
3940 */
parseTableDelimiterRow(ref OutBuffer buf,const size_t iStart,bool inQuote,ref TableColumnAlignment[]columnAlignments)3941 private size_t parseTableDelimiterRow(ref OutBuffer buf, const size_t iStart, bool inQuote, ref TableColumnAlignment[] columnAlignments)
3942 {
3943 size_t i = skipChars(buf, iStart, inQuote ? ">| \t" : "| \t");
3944 while (i < buf.length && buf[i] != '\r' && buf[i] != '\n')
3945 {
3946 const leftColon = buf[i] == ':';
3947 if (leftColon)
3948 ++i;
3949
3950 if (i >= buf.length || buf[i] != '-')
3951 break;
3952 i = skipChars(buf, i, "-");
3953
3954 const rightColon = i < buf.length && buf[i] == ':';
3955 i = skipChars(buf, i, ": \t");
3956
3957 if (i >= buf.length || (buf[i] != '|' && buf[i] != '\r' && buf[i] != '\n'))
3958 break;
3959 i = skipChars(buf, i, "| \t");
3960
3961 columnAlignments ~= (leftColon && rightColon) ? TableColumnAlignment.center :
3962 leftColon ? TableColumnAlignment.left :
3963 rightColon ? TableColumnAlignment.right :
3964 TableColumnAlignment.none;
3965 }
3966
3967 if (i < buf.length && buf[i] != '\r' && buf[i] != '\n' && buf[i] != ')')
3968 {
3969 columnAlignments.length = 0;
3970 return 0;
3971 }
3972
3973 if (i < buf.length && buf[i] == '\r') ++i;
3974 if (i < buf.length && buf[i] == '\n') ++i;
3975 return i;
3976 }
3977
3978 /****************************************************
3979 * Look for a table delimiter row, and if found parse the previous row as a
3980 * table header row. If both exist with a matching number of columns, start a
3981 * table.
3982 *
3983 * Params:
3984 * buf = an OutBuffer containing the DDoc
3985 * iStart = the index within `buf` that the table header row starts at, inclusive
3986 * iEnd = the index within `buf` that the table header row ends at, exclusive
3987 * loc = the current location in the file
3988 * inQuote = whether the table is inside a quote
3989 * inlineDelimiters = delimiters containing columns separators and any inline emphasis
3990 * columnAlignments = the parsed alignments for each column
3991 * Returns: the number of characters added by starting the table, or `0` if unchanged
3992 */
startTable(ref OutBuffer buf,size_t iStart,size_t iEnd,const ref Loc loc,bool inQuote,ref MarkdownDelimiter[]inlineDelimiters,out TableColumnAlignment[]columnAlignments)3993 private size_t startTable(ref OutBuffer buf, size_t iStart, size_t iEnd, const ref Loc loc, bool inQuote, ref MarkdownDelimiter[] inlineDelimiters, out TableColumnAlignment[] columnAlignments)
3994 {
3995 const iDelimiterRowEnd = parseTableDelimiterRow(buf, iEnd + 1, inQuote, columnAlignments);
3996 if (iDelimiterRowEnd)
3997 {
3998 size_t delta;
3999 if (replaceTableRow(buf, iStart, iEnd, loc, inlineDelimiters, columnAlignments, true, delta))
4000 {
4001 buf.remove(iEnd + delta, iDelimiterRowEnd - iEnd);
4002 buf.insert(iEnd + delta, "$(TBODY ");
4003 buf.insert(iStart, "$(TABLE ");
4004 return delta + 15;
4005 }
4006 }
4007
4008 columnAlignments.length = 0;
4009 return 0;
4010 }
4011
4012 /****************************************************
4013 * Replace a Markdown table row in the form of table cells delimited by pipes:
4014 * `| cell | cell | cell`. The first and last pipes are optional.
4015 *
4016 * Params:
4017 * buf = an OutBuffer containing the DDoc
4018 * iStart = the index within `buf` that the table row starts at, inclusive
4019 * iEnd = the index within `buf` that the table row ends at, exclusive
4020 * loc = the current location in the file
4021 * inlineDelimiters = delimiters containing columns separators and any inline emphasis
4022 * columnAlignments = alignments for each column
4023 * headerRow = if `true` then the number of columns will be enforced to match
4024 * `columnAlignments.length` and the row will be surrounded by a
4025 * `THEAD` macro
4026 * delta = the number of characters added by replacing the row, or `0` if unchanged
4027 * Returns: `true` if a table row was found and replaced
4028 */
replaceTableRow(ref OutBuffer buf,size_t iStart,size_t iEnd,const ref Loc loc,ref MarkdownDelimiter[]inlineDelimiters,TableColumnAlignment[]columnAlignments,bool headerRow,out size_t delta)4029 private bool replaceTableRow(ref OutBuffer buf, size_t iStart, size_t iEnd, const ref Loc loc, ref MarkdownDelimiter[] inlineDelimiters, TableColumnAlignment[] columnAlignments, bool headerRow, out size_t delta)
4030 {
4031 delta = 0;
4032
4033 if (!columnAlignments.length || iStart == iEnd)
4034 return false;
4035
4036 iStart = skipChars(buf, iStart, " \t");
4037 int cellCount = 0;
4038 foreach (delimiter; inlineDelimiters)
4039 if (delimiter.type == '|' && !delimiter.leftFlanking)
4040 ++cellCount;
4041 bool ignoreLast = inlineDelimiters.length > 0 && inlineDelimiters[$-1].type == '|';
4042 if (ignoreLast)
4043 {
4044 const iLast = skipChars(buf, inlineDelimiters[$-1].iStart + inlineDelimiters[$-1].count, " \t");
4045 ignoreLast = iLast >= iEnd;
4046 }
4047 if (!ignoreLast)
4048 ++cellCount;
4049
4050 if (headerRow && cellCount != columnAlignments.length)
4051 return false;
4052
4053 if (headerRow && global.params.vmarkdown)
4054 {
4055 const s = buf[][iStart..iEnd];
4056 message(loc, "Ddoc: formatting table '%.*s'", cast(int)s.length, s.ptr);
4057 }
4058
4059 void replaceTableCell(size_t iCellStart, size_t iCellEnd, int cellIndex, int di)
4060 {
4061 const eDelta = replaceMarkdownEmphasis(buf, loc, inlineDelimiters, di);
4062 delta += eDelta;
4063 iCellEnd += eDelta;
4064
4065 // strip trailing whitespace and delimiter
4066 size_t i = iCellEnd - 1;
4067 while (i > iCellStart && (buf[i] == '|' || buf[i] == ' ' || buf[i] == '\t'))
4068 --i;
4069 ++i;
4070 buf.remove(i, iCellEnd - i);
4071 delta -= iCellEnd - i;
4072 iCellEnd = i;
4073
4074 buf.insert(iCellEnd, ")");
4075 ++delta;
4076
4077 // strip initial whitespace and delimiter
4078 i = skipChars(buf, iCellStart, "| \t");
4079 buf.remove(iCellStart, i - iCellStart);
4080 delta -= i - iCellStart;
4081
4082 switch (columnAlignments[cellIndex])
4083 {
4084 case TableColumnAlignment.none:
4085 buf.insert(iCellStart, headerRow ? "$(TH " : "$(TD ");
4086 delta += 5;
4087 break;
4088 case TableColumnAlignment.left:
4089 buf.insert(iCellStart, "left, ");
4090 delta += 6;
4091 goto default;
4092 case TableColumnAlignment.center:
4093 buf.insert(iCellStart, "center, ");
4094 delta += 8;
4095 goto default;
4096 case TableColumnAlignment.right:
4097 buf.insert(iCellStart, "right, ");
4098 delta += 7;
4099 goto default;
4100 default:
4101 buf.insert(iCellStart, headerRow ? "$(TH_ALIGN " : "$(TD_ALIGN ");
4102 delta += 11;
4103 break;
4104 }
4105 }
4106
4107 int cellIndex = cellCount - 1;
4108 size_t iCellEnd = iEnd;
4109 foreach_reverse (di, delimiter; inlineDelimiters)
4110 {
4111 if (delimiter.type == '|')
4112 {
4113 if (ignoreLast && di == inlineDelimiters.length-1)
4114 {
4115 ignoreLast = false;
4116 continue;
4117 }
4118
4119 if (cellIndex >= columnAlignments.length)
4120 {
4121 // kill any extra cells
4122 buf.remove(delimiter.iStart, iEnd + delta - delimiter.iStart);
4123 delta -= iEnd + delta - delimiter.iStart;
4124 iCellEnd = iEnd + delta;
4125 --cellIndex;
4126 continue;
4127 }
4128
4129 replaceTableCell(delimiter.iStart, iCellEnd, cellIndex, cast(int) di);
4130 iCellEnd = delimiter.iStart;
4131 --cellIndex;
4132 }
4133 }
4134
4135 // if no starting pipe, replace from the start
4136 if (cellIndex >= 0)
4137 replaceTableCell(iStart, iCellEnd, cellIndex, 0);
4138
4139 buf.insert(iEnd + delta, ")");
4140 buf.insert(iStart, "$(TR ");
4141 delta += 6;
4142
4143 if (headerRow)
4144 {
4145 buf.insert(iEnd + delta, ")");
4146 buf.insert(iStart, "$(THEAD ");
4147 delta += 9;
4148 }
4149
4150 return true;
4151 }
4152
4153 /****************************************************
4154 * End a table, if in one.
4155 *
4156 * Params:
4157 * buf = an OutBuffer containing the DDoc
4158 * i = the index within `buf` to end the table at
4159 * columnAlignments = alignments for each column; upon return is set to length `0`
4160 * Returns: the number of characters added by ending the table, or `0` if unchanged
4161 */
endTable(ref OutBuffer buf,size_t i,ref TableColumnAlignment[]columnAlignments)4162 private size_t endTable(ref OutBuffer buf, size_t i, ref TableColumnAlignment[] columnAlignments)
4163 {
4164 if (!columnAlignments.length)
4165 return 0;
4166
4167 buf.insert(i, "))");
4168 columnAlignments.length = 0;
4169 return 2;
4170 }
4171
4172 /****************************************************
4173 * End a table row and then the table itself.
4174 *
4175 * Params:
4176 * buf = an OutBuffer containing the DDoc
4177 * iStart = the index within `buf` that the table row starts at, inclusive
4178 * iEnd = the index within `buf` that the table row ends at, exclusive
4179 * loc = the current location in the file
4180 * inlineDelimiters = delimiters containing columns separators and any inline emphasis
4181 * columnAlignments = alignments for each column; upon return is set to length `0`
4182 * Returns: the number of characters added by replacing the row, or `0` if unchanged
4183 */
endRowAndTable(ref OutBuffer buf,size_t iStart,size_t iEnd,const ref Loc loc,ref MarkdownDelimiter[]inlineDelimiters,ref TableColumnAlignment[]columnAlignments)4184 private size_t endRowAndTable(ref OutBuffer buf, size_t iStart, size_t iEnd, const ref Loc loc, ref MarkdownDelimiter[] inlineDelimiters, ref TableColumnAlignment[] columnAlignments)
4185 {
4186 size_t delta;
4187 replaceTableRow(buf, iStart, iEnd, loc, inlineDelimiters, columnAlignments, false, delta);
4188 delta += endTable(buf, iEnd + delta, columnAlignments);
4189 return delta;
4190 }
4191
4192 /**************************************************
4193 * Highlight text section.
4194 *
4195 * Params:
4196 * scope = the current parse scope
4197 * a = an array of D symbols at the current scope
4198 * loc = source location of start of text. It is a mutable copy to allow incrementing its linenum, for printing the correct line number when an error is encountered in a multiline block of ddoc.
4199 * buf = an OutBuffer containing the DDoc
4200 * offset = the index within buf to start highlighting
4201 */
highlightText(Scope * sc,Dsymbols * a,Loc loc,ref OutBuffer buf,size_t offset)4202 private void highlightText(Scope* sc, Dsymbols* a, Loc loc, ref OutBuffer buf, size_t offset)
4203 {
4204 const incrementLoc = loc.linnum == 0 ? 1 : 0;
4205 loc.linnum += incrementLoc;
4206 loc.charnum = 0;
4207 //printf("highlightText()\n");
4208 bool leadingBlank = true;
4209 size_t iParagraphStart = offset;
4210 size_t iPrecedingBlankLine = 0;
4211 int headingLevel = 0;
4212 int headingMacroLevel = 0;
4213 int quoteLevel = 0;
4214 bool lineQuoted = false;
4215 int quoteMacroLevel = 0;
4216 MarkdownList[] nestedLists;
4217 MarkdownDelimiter[] inlineDelimiters;
4218 MarkdownLinkReferences linkReferences;
4219 TableColumnAlignment[] columnAlignments;
4220 bool tableRowDetected = false;
4221 int inCode = 0;
4222 int inBacktick = 0;
4223 int macroLevel = 0;
4224 int previousMacroLevel = 0;
4225 int parenLevel = 0;
4226 size_t iCodeStart = 0; // start of code section
4227 size_t codeFenceLength = 0;
4228 size_t codeIndent = 0;
4229 string codeLanguage;
4230 size_t iLineStart = offset;
4231 linkReferences._scope = sc;
4232 for (size_t i = offset; i < buf.length; i++)
4233 {
4234 char c = buf[i];
4235 Lcont:
4236 switch (c)
4237 {
4238 case ' ':
4239 case '\t':
4240 break;
4241 case '\n':
4242 if (inBacktick)
4243 {
4244 // `inline code` is only valid if contained on a single line
4245 // otherwise, the backticks should be output literally.
4246 //
4247 // This lets things like `output from the linker' display
4248 // unmolested while keeping the feature consistent with GitHub.
4249 inBacktick = false;
4250 inCode = false; // the backtick also assumes we're in code
4251 // Nothing else is necessary since the DDOC_BACKQUOTED macro is
4252 // inserted lazily at the close quote, meaning the rest of the
4253 // text is already OK.
4254 }
4255 if (headingLevel)
4256 {
4257 i += replaceMarkdownEmphasis(buf, loc, inlineDelimiters);
4258 endMarkdownHeading(buf, iParagraphStart, i, loc, headingLevel);
4259 removeBlankLineMacro(buf, iPrecedingBlankLine, i);
4260 ++i;
4261 iParagraphStart = skipChars(buf, i, " \t\r\n");
4262 }
4263
4264 if (tableRowDetected && !columnAlignments.length)
4265 i += startTable(buf, iLineStart, i, loc, lineQuoted, inlineDelimiters, columnAlignments);
4266 else if (columnAlignments.length)
4267 {
4268 size_t delta;
4269 if (replaceTableRow(buf, iLineStart, i, loc, inlineDelimiters, columnAlignments, false, delta))
4270 i += delta;
4271 else
4272 i += endTable(buf, i, columnAlignments);
4273 }
4274
4275 if (!inCode && nestedLists.length && !quoteLevel)
4276 MarkdownList.handleSiblingOrEndingList(buf, i, iParagraphStart, nestedLists);
4277
4278 iPrecedingBlankLine = 0;
4279 if (!inCode && i == iLineStart && i + 1 < buf.length) // if "\n\n"
4280 {
4281 i += endTable(buf, i, columnAlignments);
4282 if (!lineQuoted && quoteLevel)
4283 endAllListsAndQuotes(buf, i, nestedLists, quoteLevel, quoteMacroLevel);
4284 i += replaceMarkdownEmphasis(buf, loc, inlineDelimiters);
4285
4286 // if we don't already know about this paragraph break then
4287 // insert a blank line and record the paragraph break
4288 if (iParagraphStart <= i)
4289 {
4290 iPrecedingBlankLine = i;
4291 i = buf.insert(i, "$(DDOC_BLANKLINE)");
4292 iParagraphStart = i + 1;
4293 }
4294 }
4295 else if (inCode &&
4296 i == iLineStart &&
4297 i + 1 < buf.length &&
4298 !lineQuoted &&
4299 quoteLevel) // if "\n\n" in quoted code
4300 {
4301 inCode = false;
4302 i = buf.insert(i, ")");
4303 i += endAllMarkdownQuotes(buf, i, quoteLevel);
4304 quoteMacroLevel = 0;
4305 }
4306 leadingBlank = true;
4307 lineQuoted = false;
4308 tableRowDetected = false;
4309 iLineStart = i + 1;
4310 loc.linnum += incrementLoc;
4311
4312 // update the paragraph start if we just entered a macro
4313 if (previousMacroLevel < macroLevel && iParagraphStart < iLineStart)
4314 iParagraphStart = iLineStart;
4315 previousMacroLevel = macroLevel;
4316 break;
4317
4318 case '<':
4319 {
4320 leadingBlank = false;
4321 if (inCode)
4322 break;
4323 const slice = buf[];
4324 auto p = &slice[i];
4325 const se = sc._module.escapetable.escapeChar('<');
4326 if (se == "<")
4327 {
4328 // Generating HTML
4329 // Skip over comments
4330 if (p[1] == '!' && p[2] == '-' && p[3] == '-')
4331 {
4332 size_t j = i + 4;
4333 p += 4;
4334 while (1)
4335 {
4336 if (j == slice.length)
4337 goto L1;
4338 if (p[0] == '-' && p[1] == '-' && p[2] == '>')
4339 {
4340 i = j + 2; // place on closing '>'
4341 break;
4342 }
4343 j++;
4344 p++;
4345 }
4346 break;
4347 }
4348 // Skip over HTML tag
4349 if (isalpha(p[1]) || (p[1] == '/' && isalpha(p[2])))
4350 {
4351 size_t j = i + 2;
4352 p += 2;
4353 while (1)
4354 {
4355 if (j == slice.length)
4356 break;
4357 if (p[0] == '>')
4358 {
4359 i = j; // place on closing '>'
4360 break;
4361 }
4362 j++;
4363 p++;
4364 }
4365 break;
4366 }
4367 }
4368 L1:
4369 // Replace '<' with '<' character entity
4370 if (se.length)
4371 {
4372 buf.remove(i, 1);
4373 i = buf.insert(i, se);
4374 i--; // point to ';'
4375 }
4376 break;
4377 }
4378
4379 case '>':
4380 {
4381 if (leadingBlank && (!inCode || quoteLevel) && global.params.markdown)
4382 {
4383 if (!quoteLevel && global.params.vmarkdown)
4384 {
4385 size_t iEnd = i + 1;
4386 while (iEnd < buf.length && buf[iEnd] != '\n')
4387 ++iEnd;
4388 const s = buf[][i .. iEnd];
4389 message(loc, "Ddoc: starting quote block with '%.*s'", cast(int)s.length, s.ptr);
4390 }
4391
4392 lineQuoted = true;
4393 int lineQuoteLevel = 1;
4394 size_t iAfterDelimiters = i + 1;
4395 for (; iAfterDelimiters < buf.length; ++iAfterDelimiters)
4396 {
4397 const c0 = buf[iAfterDelimiters];
4398 if (c0 == '>')
4399 ++lineQuoteLevel;
4400 else if (c0 != ' ' && c0 != '\t')
4401 break;
4402 }
4403 if (!quoteMacroLevel)
4404 quoteMacroLevel = macroLevel;
4405 buf.remove(i, iAfterDelimiters - i);
4406
4407 if (quoteLevel < lineQuoteLevel)
4408 {
4409 i += endRowAndTable(buf, iLineStart, i, loc, inlineDelimiters, columnAlignments);
4410 if (nestedLists.length)
4411 {
4412 const indent = getMarkdownIndent(buf, iLineStart, i);
4413 if (indent < nestedLists[$-1].contentIndent)
4414 i += MarkdownList.endAllNestedLists(buf, i, nestedLists);
4415 }
4416
4417 for (; quoteLevel < lineQuoteLevel; ++quoteLevel)
4418 {
4419 i = buf.insert(i, "$(BLOCKQUOTE\n");
4420 iLineStart = iParagraphStart = i;
4421 }
4422 --i;
4423 }
4424 else
4425 {
4426 --i;
4427 if (nestedLists.length)
4428 MarkdownList.handleSiblingOrEndingList(buf, i, iParagraphStart, nestedLists);
4429 }
4430 break;
4431 }
4432
4433 leadingBlank = false;
4434 if (inCode)
4435 break;
4436 // Replace '>' with '>' character entity
4437 const se = sc._module.escapetable.escapeChar('>');
4438 if (se.length)
4439 {
4440 buf.remove(i, 1);
4441 i = buf.insert(i, se);
4442 i--; // point to ';'
4443 }
4444 break;
4445 }
4446
4447 case '&':
4448 {
4449 leadingBlank = false;
4450 if (inCode)
4451 break;
4452 char* p = cast(char*)&buf[].ptr[i];
4453 if (p[1] == '#' || isalpha(p[1]))
4454 break;
4455 // already a character entity
4456 // Replace '&' with '&' character entity
4457 const se = sc._module.escapetable.escapeChar('&');
4458 if (se)
4459 {
4460 buf.remove(i, 1);
4461 i = buf.insert(i, se);
4462 i--; // point to ';'
4463 }
4464 break;
4465 }
4466
4467 case '`':
4468 {
4469 const iAfterDelimiter = skipChars(buf, i, "`");
4470 const count = iAfterDelimiter - i;
4471
4472 if (inBacktick == count)
4473 {
4474 inBacktick = 0;
4475 inCode = 0;
4476 OutBuffer codebuf;
4477 codebuf.write(buf[iCodeStart + count .. i]);
4478 // escape the contents, but do not perform highlighting except for DDOC_PSYMBOL
4479 highlightCode(sc, a, codebuf, 0);
4480 escapeStrayParenthesis(loc, &codebuf, 0, false);
4481 buf.remove(iCodeStart, i - iCodeStart + count); // also trimming off the current `
4482 immutable pre = "$(DDOC_BACKQUOTED ";
4483 i = buf.insert(iCodeStart, pre);
4484 i = buf.insert(i, codebuf[]);
4485 i = buf.insert(i, ")");
4486 i--; // point to the ending ) so when the for loop does i++, it will see the next character
4487 break;
4488 }
4489
4490 // Perhaps we're starting or ending a Markdown code block
4491 if (leadingBlank && global.params.markdown && count >= 3)
4492 {
4493 bool moreBackticks = false;
4494 for (size_t j = iAfterDelimiter; !moreBackticks && j < buf.length; ++j)
4495 if (buf[j] == '`')
4496 moreBackticks = true;
4497 else if (buf[j] == '\r' || buf[j] == '\n')
4498 break;
4499 if (!moreBackticks)
4500 goto case '-';
4501 }
4502
4503 if (inCode)
4504 {
4505 if (inBacktick)
4506 i = iAfterDelimiter - 1;
4507 break;
4508 }
4509 inCode = c;
4510 inBacktick = cast(int) count;
4511 codeIndent = 0; // inline code is not indented
4512 // All we do here is set the code flags and record
4513 // the location. The macro will be inserted lazily
4514 // so we can easily cancel the inBacktick if we come
4515 // across a newline character.
4516 iCodeStart = i;
4517 i = iAfterDelimiter - 1;
4518 break;
4519 }
4520
4521 case '#':
4522 {
4523 /* A line beginning with # indicates an ATX-style heading. */
4524 if (leadingBlank && !inCode)
4525 {
4526 leadingBlank = false;
4527
4528 headingLevel = detectAtxHeadingLevel(buf, i);
4529 if (!headingLevel)
4530 break;
4531
4532 i += endRowAndTable(buf, iLineStart, i, loc, inlineDelimiters, columnAlignments);
4533 if (!lineQuoted && quoteLevel)
4534 i += endAllListsAndQuotes(buf, iLineStart, nestedLists, quoteLevel, quoteMacroLevel);
4535
4536 // remove the ### prefix, including whitespace
4537 i = skipChars(buf, i + headingLevel, " \t");
4538 buf.remove(iLineStart, i - iLineStart);
4539 i = iParagraphStart = iLineStart;
4540
4541 removeAnyAtxHeadingSuffix(buf, i);
4542 --i;
4543
4544 headingMacroLevel = macroLevel;
4545 }
4546 break;
4547 }
4548
4549 case '~':
4550 {
4551 if (leadingBlank && global.params.markdown)
4552 {
4553 // Perhaps we're starting or ending a Markdown code block
4554 const iAfterDelimiter = skipChars(buf, i, "~");
4555 if (iAfterDelimiter - i >= 3)
4556 goto case '-';
4557 }
4558 leadingBlank = false;
4559 break;
4560 }
4561
4562 case '-':
4563 /* A line beginning with --- delimits a code section.
4564 * inCode tells us if it is start or end of a code section.
4565 */
4566 if (leadingBlank)
4567 {
4568 if (!inCode && c == '-')
4569 {
4570 const list = MarkdownList.parseItem(buf, iLineStart, i);
4571 if (list.isValid)
4572 {
4573 if (replaceMarkdownThematicBreak(buf, i, iLineStart, loc))
4574 {
4575 removeBlankLineMacro(buf, iPrecedingBlankLine, i);
4576 iParagraphStart = skipChars(buf, i+1, " \t\r\n");
4577 break;
4578 }
4579 else
4580 goto case '+';
4581 }
4582 }
4583
4584 size_t istart = i;
4585 size_t eollen = 0;
4586 leadingBlank = false;
4587 const c0 = c; // if we jumped here from case '`' or case '~'
4588 size_t iInfoString = 0;
4589 if (!inCode)
4590 codeLanguage.length = 0;
4591 while (1)
4592 {
4593 ++i;
4594 if (i >= buf.length)
4595 break;
4596 c = buf[i];
4597 if (c == '\n')
4598 {
4599 eollen = 1;
4600 break;
4601 }
4602 if (c == '\r')
4603 {
4604 eollen = 1;
4605 if (i + 1 >= buf.length)
4606 break;
4607 if (buf[i + 1] == '\n')
4608 {
4609 eollen = 2;
4610 break;
4611 }
4612 }
4613 // BUG: handle UTF PS and LS too
4614 if (c != c0 || iInfoString)
4615 {
4616 if (global.params.markdown && !iInfoString && !inCode && i - istart >= 3)
4617 {
4618 // Start a Markdown info string, like ```ruby
4619 codeFenceLength = i - istart;
4620 i = iInfoString = skipChars(buf, i, " \t");
4621 }
4622 else if (iInfoString && c != '`')
4623 {
4624 if (!codeLanguage.length && (c == ' ' || c == '\t'))
4625 codeLanguage = cast(string) buf[iInfoString..i].idup;
4626 }
4627 else
4628 {
4629 iInfoString = 0;
4630 goto Lcont;
4631 }
4632 }
4633 }
4634 if (i - istart < 3 || (inCode && (inCode != c0 || (inCode != '-' && i - istart < codeFenceLength))))
4635 goto Lcont;
4636 if (iInfoString)
4637 {
4638 if (!codeLanguage.length)
4639 codeLanguage = cast(string) buf[iInfoString..i].idup;
4640 }
4641 else
4642 codeFenceLength = i - istart;
4643
4644 // We have the start/end of a code section
4645 // Remove the entire --- line, including blanks and \n
4646 buf.remove(iLineStart, i - iLineStart + eollen);
4647 i = iLineStart;
4648 if (eollen)
4649 leadingBlank = true;
4650 if (inCode && (i <= iCodeStart))
4651 {
4652 // Empty code section, just remove it completely.
4653 inCode = 0;
4654 break;
4655 }
4656 if (inCode)
4657 {
4658 inCode = 0;
4659 // The code section is from iCodeStart to i
4660 OutBuffer codebuf;
4661 codebuf.write(buf[iCodeStart .. i]);
4662 codebuf.writeByte(0);
4663 // Remove leading indentations from all lines
4664 bool lineStart = true;
4665 char* endp = cast(char*)codebuf[].ptr + codebuf.length;
4666 for (char* p = cast(char*)codebuf[].ptr; p < endp;)
4667 {
4668 if (lineStart)
4669 {
4670 size_t j = codeIndent;
4671 char* q = p;
4672 while (j-- > 0 && q < endp && isIndentWS(q))
4673 ++q;
4674 codebuf.remove(p - cast(char*)codebuf[].ptr, q - p);
4675 assert(cast(char*)codebuf[].ptr <= p);
4676 assert(p < cast(char*)codebuf[].ptr + codebuf.length);
4677 lineStart = false;
4678 endp = cast(char*)codebuf[].ptr + codebuf.length; // update
4679 continue;
4680 }
4681 if (*p == '\n')
4682 lineStart = true;
4683 ++p;
4684 }
4685 if (!codeLanguage.length || codeLanguage == "dlang" || codeLanguage == "d")
4686 highlightCode2(sc, a, codebuf, 0);
4687 else
4688 codebuf.remove(codebuf.length-1, 1); // remove the trailing 0 byte
4689 escapeStrayParenthesis(loc, &codebuf, 0, false);
4690 buf.remove(iCodeStart, i - iCodeStart);
4691 i = buf.insert(iCodeStart, codebuf[]);
4692 i = buf.insert(i, ")\n");
4693 i -= 2; // in next loop, c should be '\n'
4694 }
4695 else
4696 {
4697 i += endRowAndTable(buf, iLineStart, i, loc, inlineDelimiters, columnAlignments);
4698 if (!lineQuoted && quoteLevel)
4699 {
4700 const delta = endAllListsAndQuotes(buf, iLineStart, nestedLists, quoteLevel, quoteMacroLevel);
4701 i += delta;
4702 istart += delta;
4703 }
4704
4705 inCode = c0;
4706 codeIndent = istart - iLineStart; // save indent count
4707 if (codeLanguage.length && codeLanguage != "dlang" && codeLanguage != "d")
4708 {
4709 // backslash-escape
4710 for (size_t j; j < codeLanguage.length - 1; ++j)
4711 if (codeLanguage[j] == '\\' && ispunct(codeLanguage[j + 1]))
4712 codeLanguage = codeLanguage[0..j] ~ codeLanguage[j + 1..$];
4713
4714 if (global.params.vmarkdown)
4715 message(loc, "Ddoc: adding code block for language '%.*s'", cast(int)codeLanguage.length, codeLanguage.ptr);
4716
4717 i = buf.insert(i, "$(OTHER_CODE ");
4718 i = buf.insert(i, codeLanguage);
4719 i = buf.insert(i, ",");
4720 }
4721 else
4722 i = buf.insert(i, "$(D_CODE ");
4723 iCodeStart = i;
4724 i--; // place i on >
4725 leadingBlank = true;
4726 }
4727 }
4728 break;
4729
4730 case '_':
4731 {
4732 if (leadingBlank && !inCode && replaceMarkdownThematicBreak(buf, i, iLineStart, loc))
4733 {
4734 i += endRowAndTable(buf, iLineStart, i, loc, inlineDelimiters, columnAlignments);
4735 if (!lineQuoted && quoteLevel)
4736 i += endAllListsAndQuotes(buf, iLineStart, nestedLists, quoteLevel, quoteMacroLevel);
4737 removeBlankLineMacro(buf, iPrecedingBlankLine, i);
4738 iParagraphStart = skipChars(buf, i+1, " \t\r\n");
4739 break;
4740 }
4741 goto default;
4742 }
4743
4744 case '+':
4745 case '0':
4746 ..
4747 case '9':
4748 {
4749 if (leadingBlank && !inCode)
4750 {
4751 MarkdownList list = MarkdownList.parseItem(buf, iLineStart, i);
4752 if (list.isValid)
4753 {
4754 // Avoid starting a numbered list in the middle of a paragraph
4755 if (!nestedLists.length && list.orderedStart.length &&
4756 iParagraphStart < iLineStart)
4757 {
4758 i += list.orderedStart.length - 1;
4759 break;
4760 }
4761
4762 i += endRowAndTable(buf, iLineStart, i, loc, inlineDelimiters, columnAlignments);
4763 if (!lineQuoted && quoteLevel)
4764 {
4765 const delta = endAllListsAndQuotes(buf, iLineStart, nestedLists, quoteLevel, quoteMacroLevel);
4766 i += delta;
4767 list.iStart += delta;
4768 list.iContentStart += delta;
4769 }
4770
4771 list.macroLevel = macroLevel;
4772 list.startItem(buf, iLineStart, i, iPrecedingBlankLine, nestedLists, loc);
4773 break;
4774 }
4775 }
4776 leadingBlank = false;
4777 break;
4778 }
4779
4780 case '*':
4781 {
4782 if (inCode || inBacktick || !global.params.markdown)
4783 {
4784 leadingBlank = false;
4785 break;
4786 }
4787
4788 if (leadingBlank)
4789 {
4790 // Check for a thematic break
4791 if (replaceMarkdownThematicBreak(buf, i, iLineStart, loc))
4792 {
4793 i += endRowAndTable(buf, iLineStart, i, loc, inlineDelimiters, columnAlignments);
4794 if (!lineQuoted && quoteLevel)
4795 i += endAllListsAndQuotes(buf, iLineStart, nestedLists, quoteLevel, quoteMacroLevel);
4796 removeBlankLineMacro(buf, iPrecedingBlankLine, i);
4797 iParagraphStart = skipChars(buf, i+1, " \t\r\n");
4798 break;
4799 }
4800
4801 // An initial * indicates a Markdown list item
4802 const list = MarkdownList.parseItem(buf, iLineStart, i);
4803 if (list.isValid)
4804 goto case '+';
4805 }
4806
4807 // Markdown emphasis
4808 const leftC = i > offset ? buf[i-1] : '\0';
4809 size_t iAfterEmphasis = skipChars(buf, i+1, "*");
4810 const rightC = iAfterEmphasis < buf.length ? buf[iAfterEmphasis] : '\0';
4811 int count = cast(int) (iAfterEmphasis - i);
4812 const leftFlanking = (rightC != '\0' && !isspace(rightC)) && (!ispunct(rightC) || leftC == '\0' || isspace(leftC) || ispunct(leftC));
4813 const rightFlanking = (leftC != '\0' && !isspace(leftC)) && (!ispunct(leftC) || rightC == '\0' || isspace(rightC) || ispunct(rightC));
4814 auto emphasis = MarkdownDelimiter(i, count, macroLevel, leftFlanking, rightFlanking, false, c);
4815
4816 if (!emphasis.leftFlanking && !emphasis.rightFlanking)
4817 {
4818 i = iAfterEmphasis - 1;
4819 break;
4820 }
4821
4822 inlineDelimiters ~= emphasis;
4823 i += emphasis.count;
4824 --i;
4825 break;
4826 }
4827
4828 case '!':
4829 {
4830 leadingBlank = false;
4831
4832 if (inCode || !global.params.markdown)
4833 break;
4834
4835 if (i < buf.length-1 && buf[i+1] == '[')
4836 {
4837 const imageStart = MarkdownDelimiter(i, 2, macroLevel, false, false, false, c);
4838 inlineDelimiters ~= imageStart;
4839 ++i;
4840 }
4841 break;
4842 }
4843 case '[':
4844 {
4845 if (inCode || !global.params.markdown)
4846 {
4847 leadingBlank = false;
4848 break;
4849 }
4850
4851 const leftC = i > offset ? buf[i-1] : '\0';
4852 const rightFlanking = leftC != '\0' && !isspace(leftC) && !ispunct(leftC);
4853 const atParagraphStart = leadingBlank && iParagraphStart >= iLineStart;
4854 const linkStart = MarkdownDelimiter(i, 1, macroLevel, false, rightFlanking, atParagraphStart, c);
4855 inlineDelimiters ~= linkStart;
4856 leadingBlank = false;
4857 break;
4858 }
4859 case ']':
4860 {
4861 leadingBlank = false;
4862
4863 if (inCode || !global.params.markdown)
4864 break;
4865
4866 for (int d = cast(int) inlineDelimiters.length - 1; d >= 0; --d)
4867 {
4868 const delimiter = inlineDelimiters[d];
4869 if (delimiter.type == '[' || delimiter.type == '!')
4870 {
4871 if (delimiter.isValid &&
4872 MarkdownLink.replaceLink(buf, i, loc, inlineDelimiters, d, linkReferences))
4873 {
4874 // if we removed a reference link then we're at line start
4875 if (i <= delimiter.iStart)
4876 leadingBlank = true;
4877
4878 // don't nest links
4879 if (delimiter.type == '[')
4880 for (--d; d >= 0; --d)
4881 if (inlineDelimiters[d].type == '[')
4882 inlineDelimiters[d].invalidate();
4883 }
4884 else
4885 {
4886 // nothing found, so kill the delimiter
4887 inlineDelimiters = inlineDelimiters[0..d] ~ inlineDelimiters[d+1..$];
4888 }
4889 break;
4890 }
4891 }
4892 break;
4893 }
4894
4895 case '|':
4896 {
4897 if (inCode || !global.params.markdown)
4898 {
4899 leadingBlank = false;
4900 break;
4901 }
4902
4903 tableRowDetected = true;
4904 inlineDelimiters ~= MarkdownDelimiter(i, 1, macroLevel, leadingBlank, false, false, c);
4905 leadingBlank = false;
4906 break;
4907 }
4908
4909 case '\\':
4910 {
4911 leadingBlank = false;
4912 if (inCode || i+1 >= buf.length || !global.params.markdown)
4913 break;
4914
4915 /* Escape Markdown special characters */
4916 char c1 = buf[i+1];
4917 if (ispunct(c1))
4918 {
4919 if (global.params.vmarkdown)
4920 message(loc, "Ddoc: backslash-escaped %c", c1);
4921
4922 buf.remove(i, 1);
4923
4924 auto se = sc._module.escapetable.escapeChar(c1);
4925 if (!se)
4926 se = c1 == '$' ? "$(DOLLAR)" : c1 == ',' ? "$(COMMA)" : null;
4927 if (se)
4928 {
4929 buf.remove(i, 1);
4930 i = buf.insert(i, se);
4931 i--; // point to escaped char
4932 }
4933 }
4934 break;
4935 }
4936
4937 case '$':
4938 {
4939 /* Look for the start of a macro, '$(Identifier'
4940 */
4941 leadingBlank = false;
4942 if (inCode || inBacktick)
4943 break;
4944 const slice = buf[];
4945 auto p = &slice[i];
4946 if (p[1] == '(' && isIdStart(&p[2]))
4947 ++macroLevel;
4948 break;
4949 }
4950
4951 case '(':
4952 {
4953 if (!inCode && i > offset && buf[i-1] != '$')
4954 ++parenLevel;
4955 break;
4956 }
4957
4958 case ')':
4959 { /* End of macro
4960 */
4961 leadingBlank = false;
4962 if (inCode || inBacktick)
4963 break;
4964 if (parenLevel > 0)
4965 --parenLevel;
4966 else if (macroLevel)
4967 {
4968 int downToLevel = cast(int) inlineDelimiters.length;
4969 while (downToLevel > 0 && inlineDelimiters[downToLevel - 1].macroLevel >= macroLevel)
4970 --downToLevel;
4971 if (headingLevel && headingMacroLevel >= macroLevel)
4972 {
4973 endMarkdownHeading(buf, iParagraphStart, i, loc, headingLevel);
4974 removeBlankLineMacro(buf, iPrecedingBlankLine, i);
4975 }
4976 i += endRowAndTable(buf, iLineStart, i, loc, inlineDelimiters, columnAlignments);
4977 while (nestedLists.length && nestedLists[$-1].macroLevel >= macroLevel)
4978 {
4979 i = buf.insert(i, ")\n)");
4980 --nestedLists.length;
4981 }
4982 if (quoteLevel && quoteMacroLevel >= macroLevel)
4983 i += endAllMarkdownQuotes(buf, i, quoteLevel);
4984 i += replaceMarkdownEmphasis(buf, loc, inlineDelimiters, downToLevel);
4985
4986 --macroLevel;
4987 quoteMacroLevel = 0;
4988 }
4989 break;
4990 }
4991
4992 default:
4993 leadingBlank = false;
4994 if (sc._module.filetype == FileType.ddoc || inCode)
4995 break;
4996 const start = cast(char*)buf[].ptr + i;
4997 if (isIdStart(start))
4998 {
4999 size_t j = skippastident(buf, i);
5000 if (i < j)
5001 {
5002 size_t k = skippastURL(buf, i);
5003 if (i < k)
5004 {
5005 /* The URL is buf[i..k]
5006 */
5007 if (macroLevel)
5008 /* Leave alone if already in a macro
5009 */
5010 i = k - 1;
5011 else
5012 {
5013 /* Replace URL with '$(DDOC_LINK_AUTODETECT URL)'
5014 */
5015 i = buf.bracket(i, "$(DDOC_LINK_AUTODETECT ", k, ")") - 1;
5016 }
5017 break;
5018 }
5019 }
5020 else
5021 break;
5022 size_t len = j - i;
5023 // leading '_' means no highlight unless it's a reserved symbol name
5024 if (c == '_' && (i == 0 || !isdigit(*(start - 1))) && (i == buf.length - 1 || !isReservedName(start[0 .. len])))
5025 {
5026 buf.remove(i, 1);
5027 i = buf.bracket(i, "$(DDOC_AUTO_PSYMBOL_SUPPRESS ", j - 1, ")") - 1;
5028 break;
5029 }
5030 if (isIdentifier(a, start, len))
5031 {
5032 i = buf.bracket(i, "$(DDOC_AUTO_PSYMBOL ", j, ")") - 1;
5033 break;
5034 }
5035 if (isKeyword(start, len))
5036 {
5037 i = buf.bracket(i, "$(DDOC_AUTO_KEYWORD ", j, ")") - 1;
5038 break;
5039 }
5040 if (isFunctionParameter(a, start, len))
5041 {
5042 //printf("highlighting arg '%s', i = %d, j = %d\n", arg.ident.toChars(), i, j);
5043 i = buf.bracket(i, "$(DDOC_AUTO_PARAM ", j, ")") - 1;
5044 break;
5045 }
5046 i = j - 1;
5047 }
5048 break;
5049 }
5050 }
5051
5052 if (inCode == '-')
5053 error(loc, "unmatched `---` in DDoc comment");
5054 else if (inCode)
5055 buf.insert(buf.length, ")");
5056
5057 size_t i = buf.length;
5058 if (headingLevel)
5059 {
5060 endMarkdownHeading(buf, iParagraphStart, i, loc, headingLevel);
5061 removeBlankLineMacro(buf, iPrecedingBlankLine, i);
5062 }
5063 i += endRowAndTable(buf, iLineStart, i, loc, inlineDelimiters, columnAlignments);
5064 i += replaceMarkdownEmphasis(buf, loc, inlineDelimiters);
5065 endAllListsAndQuotes(buf, i, nestedLists, quoteLevel, quoteMacroLevel);
5066 }
5067
5068 /**************************************************
5069 * Highlight code for DDOC section.
5070 */
highlightCode(Scope * sc,Dsymbol s,ref OutBuffer buf,size_t offset)5071 private void highlightCode(Scope* sc, Dsymbol s, ref OutBuffer buf, size_t offset)
5072 {
5073 auto imp = s.isImport();
5074 if (imp && imp.aliases.dim > 0)
5075 {
5076 // For example: `public import core.stdc.string : memcpy, memcmp;`
5077 for(int i = 0; i < imp.aliases.dim; i++)
5078 {
5079 // Need to distinguish between
5080 // `public import core.stdc.string : memcpy, memcmp;` and
5081 // `public import core.stdc.string : copy = memcpy, compare = memcmp;`
5082 auto a = imp.aliases[i];
5083 auto id = a ? a : imp.names[i];
5084 auto loc = Loc.init;
5085 if (auto symFromId = sc.search(loc, id, null))
5086 {
5087 highlightCode(sc, symFromId, buf, offset);
5088 }
5089 }
5090 }
5091 else
5092 {
5093 OutBuffer ancbuf;
5094 emitAnchor(ancbuf, s, sc);
5095 buf.insert(offset, ancbuf[]);
5096 offset += ancbuf.length;
5097
5098 Dsymbols a;
5099 a.push(s);
5100 highlightCode(sc, &a, buf, offset);
5101 }
5102 }
5103
5104 /****************************************************
5105 */
highlightCode(Scope * sc,Dsymbols * a,ref OutBuffer buf,size_t offset)5106 private void highlightCode(Scope* sc, Dsymbols* a, ref OutBuffer buf, size_t offset)
5107 {
5108 //printf("highlightCode(a = '%s')\n", a.toChars());
5109 bool resolvedTemplateParameters = false;
5110
5111 for (size_t i = offset; i < buf.length; i++)
5112 {
5113 char c = buf[i];
5114 const se = sc._module.escapetable.escapeChar(c);
5115 if (se.length)
5116 {
5117 buf.remove(i, 1);
5118 i = buf.insert(i, se);
5119 i--; // point to ';'
5120 continue;
5121 }
5122 char* start = cast(char*)buf[].ptr + i;
5123 if (isIdStart(start))
5124 {
5125 size_t j = skipPastIdentWithDots(buf, i);
5126 if (i < j)
5127 {
5128 size_t len = j - i;
5129 if (isIdentifier(a, start, len))
5130 {
5131 i = buf.bracket(i, "$(DDOC_PSYMBOL ", j, ")") - 1;
5132 continue;
5133 }
5134 }
5135
5136 j = skippastident(buf, i);
5137 if (i < j)
5138 {
5139 size_t len = j - i;
5140 if (isIdentifier(a, start, len))
5141 {
5142 i = buf.bracket(i, "$(DDOC_PSYMBOL ", j, ")") - 1;
5143 continue;
5144 }
5145 if (isFunctionParameter(a, start, len))
5146 {
5147 //printf("highlighting arg '%s', i = %d, j = %d\n", arg.ident.toChars(), i, j);
5148 i = buf.bracket(i, "$(DDOC_PARAM ", j, ")") - 1;
5149 continue;
5150 }
5151 i = j - 1;
5152 }
5153 }
5154 else if (!resolvedTemplateParameters)
5155 {
5156 size_t previ = i;
5157
5158 // hunt for template declarations:
5159 foreach (symi; 0 .. a.dim)
5160 {
5161 FuncDeclaration fd = (*a)[symi].isFuncDeclaration();
5162
5163 if (!fd || !fd.parent || !fd.parent.isTemplateDeclaration())
5164 {
5165 continue;
5166 }
5167
5168 TemplateDeclaration td = fd.parent.isTemplateDeclaration();
5169
5170 // build the template parameters
5171 Array!(size_t) paramLens;
5172 paramLens.reserve(td.parameters.dim);
5173
5174 OutBuffer parametersBuf;
5175 HdrGenState hgs;
5176
5177 parametersBuf.writeByte('(');
5178
5179 foreach (parami; 0 .. td.parameters.dim)
5180 {
5181 TemplateParameter tp = (*td.parameters)[parami];
5182
5183 if (parami)
5184 parametersBuf.writestring(", ");
5185
5186 size_t lastOffset = parametersBuf.length;
5187
5188 .toCBuffer(tp, ¶metersBuf, &hgs);
5189
5190 paramLens[parami] = parametersBuf.length - lastOffset;
5191 }
5192 parametersBuf.writeByte(')');
5193
5194 const templateParams = parametersBuf[];
5195
5196 //printf("templateDecl: %s\ntemplateParams: %s\nstart: %s\n", td.toChars(), templateParams, start);
5197 if (start[0 .. templateParams.length] == templateParams)
5198 {
5199 immutable templateParamListMacro = "$(DDOC_TEMPLATE_PARAM_LIST ";
5200 buf.bracket(i, templateParamListMacro.ptr, i + templateParams.length, ")");
5201
5202 // We have the parameter list. While we're here we might
5203 // as well wrap the parameters themselves as well
5204
5205 // + 1 here to take into account the opening paren of the
5206 // template param list
5207 i += templateParamListMacro.length + 1;
5208
5209 foreach (const len; paramLens)
5210 {
5211 i = buf.bracket(i, "$(DDOC_TEMPLATE_PARAM ", i + len, ")");
5212 // increment two here for space + comma
5213 i += 2;
5214 }
5215
5216 resolvedTemplateParameters = true;
5217 // reset i to be positioned back before we found the template
5218 // param list this assures that anything within the template
5219 // param list that needs to be escaped or otherwise altered
5220 // has an opportunity for that to happen outside of this context
5221 i = previ;
5222
5223 continue;
5224 }
5225 }
5226 }
5227 }
5228 }
5229
5230 /****************************************
5231 */
highlightCode3(Scope * sc,ref OutBuffer buf,const (char)* p,const (char)* pend)5232 private void highlightCode3(Scope* sc, ref OutBuffer buf, const(char)* p, const(char)* pend)
5233 {
5234 for (; p < pend; p++)
5235 {
5236 const se = sc._module.escapetable.escapeChar(*p);
5237 if (se.length)
5238 buf.writestring(se);
5239 else
5240 buf.writeByte(*p);
5241 }
5242 }
5243
5244 /**************************************************
5245 * Highlight code for CODE section.
5246 */
highlightCode2(Scope * sc,Dsymbols * a,ref OutBuffer buf,size_t offset)5247 private void highlightCode2(Scope* sc, Dsymbols* a, ref OutBuffer buf, size_t offset)
5248 {
5249 uint errorsave = global.startGagging();
5250
5251 scope Lexer lex = new Lexer(null, cast(char*)buf[].ptr, 0, buf.length - 1, 0, 1);
5252 OutBuffer res;
5253 const(char)* lastp = cast(char*)buf[].ptr;
5254 //printf("highlightCode2('%.*s')\n", cast(int)(buf.length - 1), buf[].ptr);
5255 res.reserve(buf.length);
5256 while (1)
5257 {
5258 Token tok;
5259 lex.scan(&tok);
5260 highlightCode3(sc, res, lastp, tok.ptr);
5261 string highlight = null;
5262 switch (tok.value)
5263 {
5264 case TOK.identifier:
5265 {
5266 if (!sc)
5267 break;
5268 size_t len = lex.p - tok.ptr;
5269 if (isIdentifier(a, tok.ptr, len))
5270 {
5271 highlight = "$(D_PSYMBOL ";
5272 break;
5273 }
5274 if (isFunctionParameter(a, tok.ptr, len))
5275 {
5276 //printf("highlighting arg '%s', i = %d, j = %d\n", arg.ident.toChars(), i, j);
5277 highlight = "$(D_PARAM ";
5278 break;
5279 }
5280 break;
5281 }
5282 case TOK.comment:
5283 highlight = "$(D_COMMENT ";
5284 break;
5285 case TOK.string_:
5286 highlight = "$(D_STRING ";
5287 break;
5288 default:
5289 if (tok.isKeyword())
5290 highlight = "$(D_KEYWORD ";
5291 break;
5292 }
5293 if (highlight)
5294 {
5295 res.writestring(highlight);
5296 size_t o = res.length;
5297 highlightCode3(sc, res, tok.ptr, lex.p);
5298 if (tok.value == TOK.comment || tok.value == TOK.string_)
5299 /* https://issues.dlang.org/show_bug.cgi?id=7656
5300 * https://issues.dlang.org/show_bug.cgi?id=7715
5301 * https://issues.dlang.org/show_bug.cgi?id=10519
5302 */
5303 escapeDdocString(&res, o);
5304 res.writeByte(')');
5305 }
5306 else
5307 highlightCode3(sc, res, tok.ptr, lex.p);
5308 if (tok.value == TOK.endOfFile)
5309 break;
5310 lastp = lex.p;
5311 }
5312 buf.setsize(offset);
5313 buf.write(&res);
5314 global.endGagging(errorsave);
5315 }
5316
5317 /****************************************
5318 * Determine if p points to the start of a "..." parameter identifier.
5319 */
isCVariadicArg(const (char)[]p)5320 private bool isCVariadicArg(const(char)[] p)
5321 {
5322 return p.length >= 3 && p[0 .. 3] == "...";
5323 }
5324
5325 /****************************************
5326 * Determine if p points to the start of an identifier.
5327 */
isIdStart(const (char)* p)5328 bool isIdStart(const(char)* p)
5329 {
5330 dchar c = *p;
5331 if (isalpha(c) || c == '_')
5332 return true;
5333 if (c >= 0x80)
5334 {
5335 size_t i = 0;
5336 if (utf_decodeChar(p[0 .. 4], i, c))
5337 return false; // ignore errors
5338 if (isUniAlpha(c))
5339 return true;
5340 }
5341 return false;
5342 }
5343
5344 /****************************************
5345 * Determine if p points to the rest of an identifier.
5346 */
isIdTail(const (char)* p)5347 bool isIdTail(const(char)* p)
5348 {
5349 dchar c = *p;
5350 if (isalnum(c) || c == '_')
5351 return true;
5352 if (c >= 0x80)
5353 {
5354 size_t i = 0;
5355 if (utf_decodeChar(p[0 .. 4], i, c))
5356 return false; // ignore errors
5357 if (isUniAlpha(c))
5358 return true;
5359 }
5360 return false;
5361 }
5362
5363 /****************************************
5364 * Determine if p points to the indentation space.
5365 */
isIndentWS(const (char)* p)5366 private bool isIndentWS(const(char)* p)
5367 {
5368 return (*p == ' ') || (*p == '\t');
5369 }
5370
5371 /*****************************************
5372 * Return number of bytes in UTF character.
5373 */
utfStride(const (char)* p)5374 int utfStride(const(char)* p)
5375 {
5376 dchar c = *p;
5377 if (c < 0x80)
5378 return 1;
5379 size_t i = 0;
5380 utf_decodeChar(p[0 .. 4], i, c); // ignore errors, but still consume input
5381 return cast(int)i;
5382 }
5383
inout(char)5384 private inout(char)* stripLeadingNewlines(inout(char)* s)
5385 {
5386 while (s && *s == '\n' || *s == '\r')
5387 s++;
5388
5389 return s;
5390 }
5391