xref: /llvm-project/clang/tools/scan-view/share/ScanView.py (revision dc7d7e3fe81b94e4370d1f83c758890f5d49eccd) 
1try:
2    from http.server import HTTPServer, SimpleHTTPRequestHandler
3except ImportError:
4    from BaseHTTPServer import HTTPServer
5    from SimpleHTTPServer import SimpleHTTPRequestHandler
6import os
7import sys
8import urllib, urlparse
9import posixpath
10import StringIO
11import re
12import shutil
13import threading
14import time
15import socket
16import itertools
17
18import Reporter
19import ConfigParser
20
21###
22# Various patterns matched or replaced by server.
23
24kReportFileRE = re.compile('(.*/)?report-(.*)\\.html')
25
26kBugKeyValueRE = re.compile('<!-- BUG([^ ]*) (.*) -->')
27
28#  <!-- REPORTPROBLEM file="crashes/clang_crash_ndSGF9.mi" stderr="crashes/clang_crash_ndSGF9.mi.stderr.txt" info="crashes/clang_crash_ndSGF9.mi.info" -->
29
30kReportCrashEntryRE = re.compile('<!-- REPORTPROBLEM (.*?)-->')
31kReportCrashEntryKeyValueRE = re.compile(' ?([^=]+)="(.*?)"')
32
33kReportReplacements = []
34
35# Add custom javascript.
36kReportReplacements.append((re.compile('<!-- SUMMARYENDHEAD -->'), """\
37<script language="javascript" type="text/javascript">
38function load(url) {
39  if (window.XMLHttpRequest) {
40    req = new XMLHttpRequest();
41  } else if (window.ActiveXObject) {
42    req = new ActiveXObject("Microsoft.XMLHTTP");
43  }
44  if (req != undefined) {
45    req.open("GET", url, true);
46    req.send("");
47  }
48}
49</script>"""))
50
51# Insert additional columns.
52kReportReplacements.append((re.compile('<!-- REPORTBUGCOL -->'),
53                            '<td></td><td></td>'))
54
55# Insert report bug and open file links.
56kReportReplacements.append((re.compile('<!-- REPORTBUG id="report-(.*)\\.html" -->'),
57                            ('<td class="Button"><a href="report/\\1">Report Bug</a></td>' +
58                             '<td class="Button"><a href="javascript:load(\'open/\\1\')">Open File</a></td>')))
59
60kReportReplacements.append((re.compile('<!-- REPORTHEADER -->'),
61                                       '<h3><a href="/">Summary</a> > Report %(report)s</h3>'))
62
63kReportReplacements.append((re.compile('<!-- REPORTSUMMARYEXTRA -->'),
64                            '<td class="Button"><a href="report/%(report)s">Report Bug</a></td>'))
65
66# Insert report crashes link.
67
68# Disabled for the time being until we decide exactly when this should
69# be enabled. Also the radar reporter needs to be fixed to report
70# multiple files.
71
72#kReportReplacements.append((re.compile('<!-- REPORTCRASHES -->'),
73#                            '<br>These files will automatically be attached to ' +
74#                            'reports filed here: <a href="report_crashes">Report Crashes</a>.'))
75
76###
77# Other simple parameters
78
79kShare = posixpath.join(posixpath.dirname(__file__), '../share/scan-view')
80kConfigPath = os.path.expanduser('~/.scanview.cfg')
81
82###
83
84__version__ = "0.1"
85
86__all__ = ["create_server"]
87
88class ReporterThread(threading.Thread):
89    def __init__(self, report, reporter, parameters, server):
90        threading.Thread.__init__(self)
91        self.report = report
92        self.server = server
93        self.reporter = reporter
94        self.parameters = parameters
95        self.success = False
96        self.status = None
97
98    def run(self):
99        result = None
100        try:
101            if self.server.options.debug:
102                print >>sys.stderr, "%s: SERVER: submitting bug."%(sys.argv[0],)
103            self.status = self.reporter.fileReport(self.report, self.parameters)
104            self.success = True
105            time.sleep(3)
106            if self.server.options.debug:
107                print >>sys.stderr, "%s: SERVER: submission complete."%(sys.argv[0],)
108        except Reporter.ReportFailure as e:
109            self.status = e.value
110        except Exception as e:
111            s = StringIO.StringIO()
112            import traceback
113            print >>s,'<b>Unhandled Exception</b><br><pre>'
114            traceback.print_exc(e,file=s)
115            print >>s,'</pre>'
116            self.status = s.getvalue()
117
118class ScanViewServer(HTTPServer):
119    def __init__(self, address, handler, root, reporters, options):
120        HTTPServer.__init__(self, address, handler)
121        self.root = root
122        self.reporters = reporters
123        self.options = options
124        self.halted = False
125        self.config = None
126        self.load_config()
127
128    def load_config(self):
129        self.config = ConfigParser.RawConfigParser()
130
131        # Add defaults
132        self.config.add_section('ScanView')
133        for r in self.reporters:
134            self.config.add_section(r.getName())
135            for p in r.getParameters():
136              if p.saveConfigValue():
137                self.config.set(r.getName(), p.getName(), '')
138
139        # Ignore parse errors
140        try:
141            self.config.read([kConfigPath])
142        except:
143            pass
144
145        # Save on exit
146        import atexit
147        atexit.register(lambda: self.save_config())
148
149    def save_config(self):
150        # Ignore errors (only called on exit).
151        try:
152            f = open(kConfigPath,'w')
153            self.config.write(f)
154            f.close()
155        except:
156            pass
157
158    def halt(self):
159        self.halted = True
160        if self.options.debug:
161            print >>sys.stderr, "%s: SERVER: halting." % (sys.argv[0],)
162
163    def serve_forever(self):
164        while not self.halted:
165            if self.options.debug > 1:
166                print >>sys.stderr, "%s: SERVER: waiting..." % (sys.argv[0],)
167            try:
168                self.handle_request()
169            except OSError as e:
170                print 'OSError',e.errno
171
172    def finish_request(self, request, client_address):
173        if self.options.autoReload:
174            import ScanView
175            self.RequestHandlerClass = reload(ScanView).ScanViewRequestHandler
176        HTTPServer.finish_request(self, request, client_address)
177
178    def handle_error(self, request, client_address):
179        # Ignore socket errors
180        info = sys.exc_info()
181        if info and isinstance(info[1], socket.error):
182            if self.options.debug > 1:
183                print >>sys.stderr, "%s: SERVER: ignored socket error." % (sys.argv[0],)
184            return
185        HTTPServer.handle_error(self, request, client_address)
186
187# Borrowed from Quixote, with simplifications.
188def parse_query(qs, fields=None):
189    if fields is None:
190        fields = {}
191    for chunk in filter(None, qs.split('&')):
192        if '=' not in chunk:
193            name = chunk
194            value = ''
195        else:
196            name, value = chunk.split('=', 1)
197        name = urllib.unquote(name.replace('+', ' '))
198        value = urllib.unquote(value.replace('+', ' '))
199        item = fields.get(name)
200        if item is None:
201            fields[name] = [value]
202        else:
203            item.append(value)
204    return fields
205
206class ScanViewRequestHandler(SimpleHTTPRequestHandler):
207    server_version = "ScanViewServer/" + __version__
208    dynamic_mtime = time.time()
209
210    def do_HEAD(self):
211        try:
212            SimpleHTTPRequestHandler.do_HEAD(self)
213        except Exception as e:
214            self.handle_exception(e)
215
216    def do_GET(self):
217        try:
218            SimpleHTTPRequestHandler.do_GET(self)
219        except Exception as e:
220            self.handle_exception(e)
221
222    def do_POST(self):
223        """Serve a POST request."""
224        try:
225            length = self.headers.getheader('content-length') or "0"
226            try:
227                length = int(length)
228            except:
229                length = 0
230            content = self.rfile.read(length)
231            fields = parse_query(content)
232            f = self.send_head(fields)
233            if f:
234                self.copyfile(f, self.wfile)
235                f.close()
236        except Exception as e:
237            self.handle_exception(e)
238
239    def log_message(self, format, *args):
240        if self.server.options.debug:
241            sys.stderr.write("%s: SERVER: %s - - [%s] %s\n" %
242                             (sys.argv[0],
243                              self.address_string(),
244                              self.log_date_time_string(),
245                              format%args))
246
247    def load_report(self, report):
248        path = os.path.join(self.server.root, 'report-%s.html'%report)
249        data = open(path).read()
250        keys = {}
251        for item in kBugKeyValueRE.finditer(data):
252            k,v = item.groups()
253            keys[k] = v
254        return keys
255
256    def load_crashes(self):
257        path = posixpath.join(self.server.root, 'index.html')
258        data = open(path).read()
259        problems = []
260        for item in kReportCrashEntryRE.finditer(data):
261            fieldData = item.group(1)
262            fields = dict([i.groups() for i in
263                           kReportCrashEntryKeyValueRE.finditer(fieldData)])
264            problems.append(fields)
265        return problems
266
267    def handle_exception(self, exc):
268        import traceback
269        s = StringIO.StringIO()
270        print >>s, "INTERNAL ERROR\n"
271        traceback.print_exc(exc, s)
272        f = self.send_string(s.getvalue(), 'text/plain')
273        if f:
274            self.copyfile(f, self.wfile)
275            f.close()
276
277    def get_scalar_field(self, name):
278        if name in self.fields:
279            return self.fields[name][0]
280        else:
281            return None
282
283    def submit_bug(self, c):
284        title = self.get_scalar_field('title')
285        description = self.get_scalar_field('description')
286        report = self.get_scalar_field('report')
287        reporterIndex = self.get_scalar_field('reporter')
288        files = []
289        for fileID in self.fields.get('files',[]):
290            try:
291                i = int(fileID)
292            except:
293                i = None
294            if i is None or i<0 or i>=len(c.files):
295                return (False, 'Invalid file ID')
296            files.append(c.files[i])
297
298        if not title:
299            return (False, "Missing title.")
300        if not description:
301            return (False, "Missing description.")
302        try:
303            reporterIndex = int(reporterIndex)
304        except:
305            return (False, "Invalid report method.")
306
307        # Get the reporter and parameters.
308        reporter = self.server.reporters[reporterIndex]
309        parameters = {}
310        for o in reporter.getParameters():
311            name = '%s_%s'%(reporter.getName(),o.getName())
312            if name not in self.fields:
313                return (False,
314                        'Missing field "%s" for %s report method.'%(name,
315                                                                    reporter.getName()))
316            parameters[o.getName()] = self.get_scalar_field(name)
317
318        # Update config defaults.
319        if report != 'None':
320            self.server.config.set('ScanView', 'reporter', reporterIndex)
321            for o in reporter.getParameters():
322              if o.saveConfigValue():
323                name = o.getName()
324                self.server.config.set(reporter.getName(), name, parameters[name])
325
326        # Create the report.
327        bug = Reporter.BugReport(title, description, files)
328
329        # Kick off a reporting thread.
330        t = ReporterThread(bug, reporter, parameters, self.server)
331        t.start()
332
333        # Wait for thread to die...
334        while t.isAlive():
335            time.sleep(.25)
336        submitStatus = t.status
337
338        return (t.success, t.status)
339
340    def send_report_submit(self):
341        report = self.get_scalar_field('report')
342        c = self.get_report_context(report)
343        if c.reportSource is None:
344            reportingFor = "Report Crashes > "
345            fileBug = """\
346<a href="/report_crashes">File Bug</a> > """%locals()
347        else:
348            reportingFor = '<a href="/%s">Report %s</a> > ' % (c.reportSource,
349                                                                   report)
350            fileBug = '<a href="/report/%s">File Bug</a> > ' % report
351        title = self.get_scalar_field('title')
352        description = self.get_scalar_field('description')
353
354        res,message = self.submit_bug(c)
355
356        if res:
357            statusClass = 'SubmitOk'
358            statusName = 'Succeeded'
359        else:
360            statusClass = 'SubmitFail'
361            statusName = 'Failed'
362
363        result = """
364<head>
365  <title>Bug Submission</title>
366  <link rel="stylesheet" type="text/css" href="/scanview.css" />
367</head>
368<body>
369<h3>
370<a href="/">Summary</a> >
371%(reportingFor)s
372%(fileBug)s
373Submit</h3>
374<form name="form" action="">
375<table class="form">
376<tr><td>
377<table class="form_group">
378<tr>
379  <td class="form_clabel">Title:</td>
380  <td class="form_value">
381    <input type="text" name="title" size="50" value="%(title)s" disabled>
382  </td>
383</tr>
384<tr>
385  <td class="form_label">Description:</td>
386  <td class="form_value">
387<textarea rows="10" cols="80" name="description" disabled>
388%(description)s
389</textarea>
390  </td>
391</table>
392</td></tr>
393</table>
394</form>
395<h1 class="%(statusClass)s">Submission %(statusName)s</h1>
396%(message)s
397<p>
398<hr>
399<a href="/">Return to Summary</a>
400</body>
401</html>"""%locals()
402        return self.send_string(result)
403
404    def send_open_report(self, report):
405        try:
406            keys = self.load_report(report)
407        except IOError:
408            return self.send_error(400, 'Invalid report.')
409
410        file = keys.get('FILE')
411        if not file or not posixpath.exists(file):
412            return self.send_error(400, 'File does not exist: "%s"' % file)
413
414        import startfile
415        if self.server.options.debug:
416            print >>sys.stderr, '%s: SERVER: opening "%s"'%(sys.argv[0],
417                                                            file)
418
419        status = startfile.open(file)
420        if status:
421            res = 'Opened: "%s"' % file
422        else:
423            res = 'Open failed: "%s"' % file
424
425        return self.send_string(res, 'text/plain')
426
427    def get_report_context(self, report):
428        class Context(object):
429            pass
430        if report is None or report == 'None':
431            data = self.load_crashes()
432            # Don't allow empty reports.
433            if not data:
434                raise ValueError('No crashes detected!')
435            c = Context()
436            c.title = 'clang static analyzer failures'
437
438            stderrSummary = ""
439            for item in data:
440                if 'stderr' in item:
441                    path = posixpath.join(self.server.root, item['stderr'])
442                    if os.path.exists(path):
443                        lns = itertools.islice(open(path), 0, 10)
444                        stderrSummary += '%s\n--\n%s' % (item.get('src',
445                                                                  '<unknown>'),
446                                                         ''.join(lns))
447
448            c.description = """\
449The clang static analyzer failed on these inputs:
450%s
451
452STDERR Summary
453--------------
454%s
455""" % ('\n'.join([item.get('src','<unknown>') for item in data]),
456       stderrSummary)
457            c.reportSource = None
458            c.navMarkup = "Report Crashes > "
459            c.files = []
460            for item in data:
461                c.files.append(item.get('src',''))
462                c.files.append(posixpath.join(self.server.root,
463                                              item.get('file','')))
464                c.files.append(posixpath.join(self.server.root,
465                                              item.get('clangfile','')))
466                c.files.append(posixpath.join(self.server.root,
467                                              item.get('stderr','')))
468                c.files.append(posixpath.join(self.server.root,
469                                              item.get('info','')))
470            # Just in case something failed, ignore files which don't
471            # exist.
472            c.files = [f for f in c.files
473                       if os.path.exists(f) and os.path.isfile(f)]
474        else:
475            # Check that this is a valid report.
476            path = posixpath.join(self.server.root, 'report-%s.html' % report)
477            if not posixpath.exists(path):
478                raise ValueError('Invalid report ID')
479            keys = self.load_report(report)
480            c = Context()
481            c.title = keys.get('DESC','clang error (unrecognized')
482            c.description = """\
483Bug reported by the clang static analyzer.
484
485Description: %s
486File: %s
487Line: %s
488"""%(c.title, keys.get('FILE','<unknown>'), keys.get('LINE', '<unknown>'))
489            c.reportSource = 'report-%s.html' % report
490            c.navMarkup = """<a href="/%s">Report %s</a> > """ % (c.reportSource,
491                                                                  report)
492
493            c.files = [path]
494        return c
495
496    def send_report(self, report, configOverrides=None):
497        def getConfigOption(section, field):
498            if (configOverrides is not None and
499                section in configOverrides and
500                field in configOverrides[section]):
501                return configOverrides[section][field]
502            return self.server.config.get(section, field)
503
504        # report is None is used for crashes
505        try:
506            c = self.get_report_context(report)
507        except ValueError as e:
508            return self.send_error(400, e.message)
509
510        title = c.title
511        description= c.description
512        reportingFor = c.navMarkup
513        if c.reportSource is None:
514            extraIFrame = ""
515        else:
516            extraIFrame = """\
517<iframe src="/%s" width="100%%" height="40%%"
518        scrolling="auto" frameborder="1">
519  <a href="/%s">View Bug Report</a>
520</iframe>""" % (c.reportSource, c.reportSource)
521
522        reporterSelections = []
523        reporterOptions = []
524
525        try:
526            active = int(getConfigOption('ScanView','reporter'))
527        except:
528            active = 0
529        for i,r in enumerate(self.server.reporters):
530            selected = (i == active)
531            if selected:
532                selectedStr = ' selected'
533            else:
534                selectedStr = ''
535            reporterSelections.append('<option value="%d"%s>%s</option>'%(i,selectedStr,r.getName()))
536            options = '\n'.join([ o.getHTML(r,title,getConfigOption) for o in r.getParameters()])
537            display = ('none','')[selected]
538            reporterOptions.append("""\
539<tr id="%sReporterOptions" style="display:%s">
540  <td class="form_label">%s Options</td>
541  <td class="form_value">
542    <table class="form_inner_group">
543%s
544    </table>
545  </td>
546</tr>
547"""%(r.getName(),display,r.getName(),options))
548        reporterSelections = '\n'.join(reporterSelections)
549        reporterOptionsDivs = '\n'.join(reporterOptions)
550        reportersArray = '[%s]'%(','.join([repr(r.getName()) for r in self.server.reporters]))
551
552        if c.files:
553            fieldSize = min(5, len(c.files))
554            attachFileOptions = '\n'.join(["""\
555<option value="%d" selected>%s</option>""" % (i,v) for i,v in enumerate(c.files)])
556            attachFileRow = """\
557<tr>
558  <td class="form_label">Attach:</td>
559  <td class="form_value">
560<select style="width:100%%" name="files" multiple size=%d>
561%s
562</select>
563  </td>
564</tr>
565""" % (min(5, len(c.files)), attachFileOptions)
566        else:
567            attachFileRow = ""
568
569        result = """<html>
570<head>
571  <title>File Bug</title>
572  <link rel="stylesheet" type="text/css" href="/scanview.css" />
573</head>
574<script language="javascript" type="text/javascript">
575var reporters = %(reportersArray)s;
576function updateReporterOptions() {
577  index = document.getElementById('reporter').selectedIndex;
578  for (var i=0; i < reporters.length; ++i) {
579    o = document.getElementById(reporters[i] + "ReporterOptions");
580    if (i == index) {
581      o.style.display = "";
582    } else {
583      o.style.display = "none";
584    }
585  }
586}
587</script>
588<body onLoad="updateReporterOptions()">
589<h3>
590<a href="/">Summary</a> >
591%(reportingFor)s
592File Bug</h3>
593<form name="form" action="/report_submit" method="post">
594<input type="hidden" name="report" value="%(report)s">
595
596<table class="form">
597<tr><td>
598<table class="form_group">
599<tr>
600  <td class="form_clabel">Title:</td>
601  <td class="form_value">
602    <input type="text" name="title" size="50" value="%(title)s">
603  </td>
604</tr>
605<tr>
606  <td class="form_label">Description:</td>
607  <td class="form_value">
608<textarea rows="10" cols="80" name="description">
609%(description)s
610</textarea>
611  </td>
612</tr>
613
614%(attachFileRow)s
615
616</table>
617<br>
618<table class="form_group">
619<tr>
620  <td class="form_clabel">Method:</td>
621  <td class="form_value">
622    <select id="reporter" name="reporter" onChange="updateReporterOptions()">
623    %(reporterSelections)s
624    </select>
625  </td>
626</tr>
627%(reporterOptionsDivs)s
628</table>
629<br>
630</td></tr>
631<tr><td class="form_submit">
632  <input align="right" type="submit" name="Submit" value="Submit">
633</td></tr>
634</table>
635</form>
636
637%(extraIFrame)s
638
639</body>
640</html>"""%locals()
641
642        return self.send_string(result)
643
644    def send_head(self, fields=None):
645        if (self.server.options.onlyServeLocal and
646            self.client_address[0] != '127.0.0.1'):
647            return self.send_error(401, 'Unauthorized host.')
648
649        if fields is None:
650            fields = {}
651        self.fields = fields
652
653        o = urlparse.urlparse(self.path)
654        self.fields = parse_query(o.query, fields)
655        path = posixpath.normpath(urllib.unquote(o.path))
656
657        # Split the components and strip the root prefix.
658        components = path.split('/')[1:]
659
660        # Special case some top-level entries.
661        if components:
662            name = components[0]
663            if len(components)==2:
664                if name=='report':
665                    return self.send_report(components[1])
666                elif name=='open':
667                    return self.send_open_report(components[1])
668            elif len(components)==1:
669                if name=='quit':
670                    self.server.halt()
671                    return self.send_string('Goodbye.', 'text/plain')
672                elif name=='report_submit':
673                    return self.send_report_submit()
674                elif name=='report_crashes':
675                    overrides = { 'ScanView' : {},
676                                  'Radar' : {},
677                                  'Email' : {} }
678                    for i,r in enumerate(self.server.reporters):
679                        if r.getName() == 'Radar':
680                            overrides['ScanView']['reporter'] = i
681                            break
682                    overrides['Radar']['Component'] = 'llvm - checker'
683                    overrides['Radar']['Component Version'] = 'X'
684                    return self.send_report(None, overrides)
685                elif name=='favicon.ico':
686                    return self.send_path(posixpath.join(kShare,'bugcatcher.ico'))
687
688        # Match directory entries.
689        if components[-1] == '':
690            components[-1] = 'index.html'
691
692        relpath = '/'.join(components)
693        path = posixpath.join(self.server.root, relpath)
694
695        if self.server.options.debug > 1:
696            print >>sys.stderr, '%s: SERVER: sending path "%s"'%(sys.argv[0],
697                                                                 path)
698        return self.send_path(path)
699
700    def send_404(self):
701        self.send_error(404, "File not found")
702        return None
703
704    def send_path(self, path):
705        # If the requested path is outside the root directory, do not open it
706        rel = os.path.abspath(path)
707        if not rel.startswith(os.path.abspath(self.server.root)):
708          return self.send_404()
709
710        ctype = self.guess_type(path)
711        if ctype.startswith('text/'):
712            # Patch file instead
713            return self.send_patched_file(path, ctype)
714        else:
715            mode = 'rb'
716        try:
717            f = open(path, mode)
718        except IOError:
719            return self.send_404()
720        return self.send_file(f, ctype)
721
722    def send_file(self, f, ctype):
723        # Patch files to add links, but skip binary files.
724        self.send_response(200)
725        self.send_header("Content-type", ctype)
726        fs = os.fstat(f.fileno())
727        self.send_header("Content-Length", str(fs[6]))
728        self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
729        self.end_headers()
730        return f
731
732    def send_string(self, s, ctype='text/html', headers=True, mtime=None):
733        if headers:
734            self.send_response(200)
735            self.send_header("Content-type", ctype)
736            self.send_header("Content-Length", str(len(s)))
737            if mtime is None:
738                mtime = self.dynamic_mtime
739            self.send_header("Last-Modified", self.date_time_string(mtime))
740            self.end_headers()
741        return StringIO.StringIO(s)
742
743    def send_patched_file(self, path, ctype):
744        # Allow a very limited set of variables. This is pretty gross.
745        variables = {}
746        variables['report'] = ''
747        m = kReportFileRE.match(path)
748        if m:
749            variables['report'] = m.group(2)
750
751        try:
752            f = open(path,'r')
753        except IOError:
754            return self.send_404()
755        fs = os.fstat(f.fileno())
756        data = f.read()
757        for a,b in kReportReplacements:
758            data = a.sub(b % variables, data)
759        return self.send_string(data, ctype, mtime=fs.st_mtime)
760
761
762def create_server(address, options, root):
763    import Reporter
764
765    reporters = Reporter.getReporters()
766
767    return ScanViewServer(address, ScanViewRequestHandler,
768                          root,
769                          reporters,
770                          options)
771