diff options
author | Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com> | 2020-02-28 01:41:00 +0000 |
---|---|---|
committer | Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com> | 2020-02-28 01:41:00 +0000 |
commit | b68bc994e93290c7697b573f8fa6a7f61c2272b3 (patch) | |
tree | 03766045b83c9e500ccc3168df3a9b3d61dd0ae0 | |
parent | 8ec17a7bcc46dbeb293ff2868173ccc151723dcf (diff) | |
parent | cabdab0a40609023cc566b065a8d76a8bcb87c7e (diff) | |
download | build-b68bc994e93290c7697b573f8fa6a7f61c2272b3.tar.gz |
Merge "Separate html/csv output functions into html_writer.py" am: cabdab0a40
Change-Id: I6903764cfe04589866b380bab5d8825f7f1013e0
-rw-r--r-- | tools/warn/html_writer.py | 673 | ||||
-rw-r--r-- | tools/warn/java_warn_patterns.py | 2 | ||||
-rw-r--r-- | tools/warn/make_warn_patterns.py | 2 | ||||
-rw-r--r-- | tools/warn/other_warn_patterns.py | 2 | ||||
-rw-r--r-- | tools/warn/tidy_warn_patterns.py | 2 | ||||
-rwxr-xr-x | tools/warn/warn_common.py | 666 |
6 files changed, 685 insertions, 662 deletions
diff --git a/tools/warn/html_writer.py b/tools/warn/html_writer.py new file mode 100644 index 0000000000..b8d3fe6f5d --- /dev/null +++ b/tools/warn/html_writer.py @@ -0,0 +1,673 @@ +# Lint as: python3 +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Emit warning messages to html or csv files.""" + +# To emit html page of warning messages: +# flags: --byproject, --url, --separator +# Old stuff for static html components: +# html_script_style: static html scripts and styles +# htmlbig: +# dump_stats, dump_html_prologue, dump_html_epilogue: +# emit_buttons: +# dump_fixed +# sort_warnings: +# emit_stats_by_project: +# all_patterns, +# findproject, classify_warning +# dump_html +# +# New dynamic HTML page's static JavaScript data: +# Some data are copied from Python to JavaScript, to generate HTML elements. +# FlagPlatform flags.platform +# FlagURL flags.url, used by 'android' +# FlagSeparator flags.separator, used by 'android' +# SeverityColors: list of colors for all severity levels +# SeverityHeaders: list of headers for all severity levels +# SeverityColumnHeaders: list of column_headers for all severity levels +# ProjectNames: project_names, or project_list[*][0] +# WarnPatternsSeverity: warn_patterns[*]['severity'] +# WarnPatternsDescription: warn_patterns[*]['description'] +# WarningMessages: warning_messages +# Warnings: warning_records +# StatsHeader: warning count table header row +# StatsRows: array of warning count table rows +# +# New dynamic HTML page's dynamic JavaScript data: +# +# New dynamic HTML related function to emit data: +# escape_string, strip_escape_string, emit_warning_arrays +# emit_js_data(): + +from __future__ import print_function +import cgi +import csv +import sys + +# pylint:disable=relative-beyond-top-level +# pylint:disable=g-importing-member +from .severity import Severity + + +html_head_scripts = """\ + <script type="text/javascript"> + function expand(id) { + var e = document.getElementById(id); + var f = document.getElementById(id + "_mark"); + if (e.style.display == 'block') { + e.style.display = 'none'; + f.innerHTML = '⊕'; + } + else { + e.style.display = 'block'; + f.innerHTML = '⊖'; + } + }; + function expandCollapse(show) { + for (var id = 1; ; id++) { + var e = document.getElementById(id + ""); + var f = document.getElementById(id + "_mark"); + if (!e || !f) break; + e.style.display = (show ? 'block' : 'none'); + f.innerHTML = (show ? '⊖' : '⊕'); + } + }; + </script> + <style type="text/css"> + th,td{border-collapse:collapse; border:1px solid black;} + .button{color:blue;font-size:110%;font-weight:bolder;} + .bt{color:black;background-color:transparent;border:none;outline:none; + font-size:140%;font-weight:bolder;} + .c0{background-color:#e0e0e0;} + .c1{background-color:#d0d0d0;} + .t1{border-collapse:collapse; width:100%; border:1px solid black;} + </style> + <script src="https://www.gstatic.com/charts/loader.js"></script> +""" + + +def make_writer(output_stream): + + def writer(text): + return output_stream.write(text + '\n') + + return writer + + +def html_big(param): + return '<font size="+2">' + param + '</font>' + + +def dump_html_prologue(title, writer, warn_patterns, project_names): + writer('<html>\n<head>') + writer('<title>' + title + '</title>') + writer(html_head_scripts) + emit_stats_by_project(writer, warn_patterns, project_names) + writer('</head>\n<body>') + writer(html_big(title)) + writer('<p>') + + +def dump_html_epilogue(writer): + writer('</body>\n</head>\n</html>') + + +def sort_warnings(warn_patterns): + for i in warn_patterns: + i['members'] = sorted(set(i['members'])) + + +def create_warnings(warn_patterns, project_names): + """Creates warnings s.t. + + warnings[p][s] is as specified in above docs. + + Args: + warn_patterns: list of warning patterns for specified platform + project_names: list of project names + + Returns: + 2D warnings array where warnings[p][s] is # of warnings in project name p of + severity level s + """ + # pylint:disable=g-complex-comprehension + warnings = {p: {s.value: 0 for s in Severity.levels} for p in project_names} + for i in warn_patterns: + s = i['severity'].value + for p in i['projects']: + warnings[p][s] += i['projects'][p] + return warnings + + +def get_total_by_project(warnings, project_names): + """Returns dict, project as key and # warnings for that project as value.""" + # pylint:disable=g-complex-comprehension + return { + p: sum(warnings[p][s.value] for s in Severity.levels) + for p in project_names + } + + +def get_total_by_severity(warnings, project_names): + """Returns dict, severity as key and # warnings of that severity as value.""" + # pylint:disable=g-complex-comprehension + return { + s.value: sum(warnings[p][s.value] for p in project_names) + for s in Severity.levels + } + + +def emit_table_header(total_by_severity): + """Returns list of HTML-formatted content for severity stats.""" + + stats_header = ['Project'] + for s in Severity.levels: + if total_by_severity[s.value]: + stats_header.append( + '<span style=\'background-color:{}\'>{}</span>'.format( + s.color, s.column_header)) + stats_header.append('TOTAL') + return stats_header + + +def emit_row_counts_per_project(warnings, total_by_project, total_by_severity, + project_names): + """Returns total project warnings and row of stats for each project. + + Args: + warnings: output of create_warnings(warn_patterns, project_names) + total_by_project: output of get_total_by_project(project_names) + total_by_severity: output of get_total_by_severity(project_names) + project_names: list of project names + + Returns: + total_all_projects, the total number of warnings over all projects + stats_rows, a 2d list where each row is [Project Name, <severity counts>, + total # warnings for this project] + """ + + total_all_projects = 0 + stats_rows = [] + for p in project_names: + if total_by_project[p]: + one_row = [p] + for s in Severity.levels: + if total_by_severity[s.value]: + one_row.append(warnings[p][s.value]) + one_row.append(total_by_project[p]) + stats_rows.append(one_row) + total_all_projects += total_by_project[p] + return total_all_projects, stats_rows + + +def emit_row_counts_per_severity(total_by_severity, stats_header, stats_rows, + total_all_projects, writer): + """Emits stats_header and stats_rows as specified above. + + Args: + total_by_severity: output of get_total_by_severity() + stats_header: output of emit_table_header() + stats_rows: output of emit_row_counts_per_project() + total_all_projects: output of emit_row_counts_per_project() + writer: writer returned by make_writer(output_stream) + """ + + total_all_severities = 0 + one_row = ['<b>TOTAL</b>'] + for s in Severity.levels: + if total_by_severity[s.value]: + one_row.append(total_by_severity[s.value]) + total_all_severities += total_by_severity[s.value] + one_row.append(total_all_projects) + stats_rows.append(one_row) + writer('<script>') + emit_const_string_array('StatsHeader', stats_header, writer) + emit_const_object_array('StatsRows', stats_rows, writer) + writer(draw_table_javascript) + writer('</script>') + + +def emit_stats_by_project(writer, warn_patterns, project_names): + """Dump a google chart table of warnings per project and severity.""" + + warnings = create_warnings(warn_patterns, project_names) + total_by_project = get_total_by_project(warnings, project_names) + total_by_severity = get_total_by_severity(warnings, project_names) + stats_header = emit_table_header(total_by_severity) + total_all_projects, stats_rows = \ + emit_row_counts_per_project(warnings, total_by_project, total_by_severity, project_names) + emit_row_counts_per_severity(total_by_severity, stats_header, stats_rows, + total_all_projects, writer) + + +def dump_stats(writer, warn_patterns): + """Dump some stats about total number of warnings and such.""" + + known = 0 + skipped = 0 + unknown = 0 + sort_warnings(warn_patterns) + for i in warn_patterns: + if i['severity'] == Severity.UNMATCHED: + unknown += len(i['members']) + elif i['severity'] == Severity.SKIP: + skipped += len(i['members']) + else: + known += len(i['members']) + writer('Number of classified warnings: <b>' + str(known) + '</b><br>') + writer('Number of skipped warnings: <b>' + str(skipped) + '</b><br>') + writer('Number of unclassified warnings: <b>' + str(unknown) + '</b><br>') + total = unknown + known + skipped + extra_msg = '' + if total < 1000: + extra_msg = ' (low count may indicate incremental build)' + writer('Total number of warnings: <b>' + str(total) + '</b>' + extra_msg) + + +# New base table of warnings, [severity, warn_id, project, warning_message] +# Need buttons to show warnings in different grouping options. +# (1) Current, group by severity, id for each warning pattern +# sort by severity, warn_id, warning_message +# (2) Current --byproject, group by severity, +# id for each warning pattern + project name +# sort by severity, warn_id, project, warning_message +# (3) New, group by project + severity, +# id for each warning pattern +# sort by project, severity, warn_id, warning_message +def emit_buttons(writer): + writer('<button class="button" onclick="expandCollapse(1);">' + 'Expand all warnings</button>\n' + '<button class="button" onclick="expandCollapse(0);">' + 'Collapse all warnings</button>\n' + '<button class="button" onclick="groupBySeverity();">' + 'Group warnings by severity</button>\n' + '<button class="button" onclick="groupByProject();">' + 'Group warnings by project</button><br>') + + +def all_patterns(category): + patterns = '' + for i in category['patterns']: + patterns += i + patterns += ' / ' + return patterns + + +def dump_fixed(writer, warn_patterns): + """Show which warnings no longer occur.""" + anchor = 'fixed_warnings' + mark = anchor + '_mark' + writer('\n<br><p style="background-color:lightblue"><b>' + '<button id="' + mark + '" ' + 'class="bt" onclick="expand(\'' + anchor + '\');">' + '⊕</button> Fixed warnings. ' + 'No more occurrences. Please consider turning these into ' + 'errors if possible, before they are reintroduced in to the build' + ':</b></p>') + writer('<blockquote>') + fixed_patterns = [] + for i in warn_patterns: + if not i['members']: + fixed_patterns.append(i['description'] + ' (' + all_patterns(i) + ')') + fixed_patterns = sorted(fixed_patterns) + writer('<div id="' + anchor + '" style="display:none;"><table>') + cur_row_class = 0 + for text in fixed_patterns: + cur_row_class = 1 - cur_row_class + # remove last '\n' + t = text[:-1] if text[-1] == '\n' else text + writer('<tr><td class="c' + str(cur_row_class) + '">' + t + '</td></tr>') + writer('</table></div>') + writer('</blockquote>') + + +def write_severity(csvwriter, sev, kind, warn_patterns): + """Count warnings of given severity and write CSV entries to writer.""" + total = 0 + for pattern in warn_patterns: + if pattern['severity'] == sev and pattern['members']: + n = len(pattern['members']) + total += n + warning = kind + ': ' + (pattern['description'] or '?') + csvwriter.writerow([n, '', warning]) + # print number of warnings for each project, ordered by project name + projects = sorted(pattern['projects'].keys()) + for project in projects: + csvwriter.writerow([pattern['projects'][project], project, warning]) + csvwriter.writerow([total, '', kind + ' warnings']) + return total + + +def dump_csv(csvwriter, warn_patterns): + """Dump number of warnings in CSV format to writer.""" + sort_warnings(warn_patterns) + total = 0 + for s in Severity.levels: + total += write_severity(csvwriter, s, s.column_header, warn_patterns) + csvwriter.writerow([total, '', 'All warnings']) + + +# Return s with escaped backslash and quotation characters. +def escape_string(s): + return s.replace('\\', '\\\\').replace('"', '\\"') + + +# Return s without trailing '\n' and escape the quotation characters. +def strip_escape_string(s): + if not s: + return s + s = s[:-1] if s[-1] == '\n' else s + return escape_string(s) + + +def emit_warning_array(name, writer, warn_patterns): + writer('var warning_{} = ['.format(name)) + for w in warn_patterns: + if name == 'severity': + writer('{},'.format(w[name].value)) + else: + writer('{},'.format(w[name])) + writer('];') + + +def emit_warning_arrays(writer, warn_patterns): + emit_warning_array('severity', writer, warn_patterns) + writer('var warning_description = [') + for w in warn_patterns: + if w['members']: + writer('"{}",'.format(escape_string(w['description']))) + else: + writer('"",') # no such warning + writer('];') + + +scripts_for_warning_groups = """ + function compareMessages(x1, x2) { // of the same warning type + return (WarningMessages[x1[2]] <= WarningMessages[x2[2]]) ? -1 : 1; + } + function byMessageCount(x1, x2) { + return x2[2] - x1[2]; // reversed order + } + function bySeverityMessageCount(x1, x2) { + // orer by severity first + if (x1[1] != x2[1]) + return x1[1] - x2[1]; + return byMessageCount(x1, x2); + } + const ParseLinePattern = /^([^ :]+):(\\d+):(.+)/; + function addURL(line) { // used by Android + if (FlagURL == "") return line; + if (FlagSeparator == "") { + return line.replace(ParseLinePattern, + "<a target='_blank' href='" + FlagURL + "/$1'>$1</a>:$2:$3"); + } + return line.replace(ParseLinePattern, + "<a target='_blank' href='" + FlagURL + "/$1" + FlagSeparator + + "$2'>$1:$2</a>:$3"); + } + function addURLToLine(line, link) { // used by Chrome + let line_split = line.split(":"); + let path = line_split.slice(0,3).join(":"); + let msg = line_split.slice(3).join(":"); + let html_link = `<a target="_blank" href="${link}">${path}</a>${msg}`; + return html_link; + } + function createArrayOfDictionaries(n) { + var result = []; + for (var i=0; i<n; i++) result.push({}); + return result; + } + function groupWarningsBySeverity() { + // groups is an array of dictionaries, + // each dictionary maps from warning type to array of warning messages. + var groups = createArrayOfDictionaries(SeverityColors.length); + for (var i=0; i<Warnings.length; i++) { + var w = Warnings[i][0]; + var s = WarnPatternsSeverity[w]; + var k = w.toString(); + if (!(k in groups[s])) + groups[s][k] = []; + groups[s][k].push(Warnings[i]); + } + return groups; + } + function groupWarningsByProject() { + var groups = createArrayOfDictionaries(ProjectNames.length); + for (var i=0; i<Warnings.length; i++) { + var w = Warnings[i][0]; + var p = Warnings[i][1]; + var k = w.toString(); + if (!(k in groups[p])) + groups[p][k] = []; + groups[p][k].push(Warnings[i]); + } + return groups; + } + var GlobalAnchor = 0; + function createWarningSection(header, color, group) { + var result = ""; + var groupKeys = []; + var totalMessages = 0; + for (var k in group) { + totalMessages += group[k].length; + groupKeys.push([k, WarnPatternsSeverity[parseInt(k)], group[k].length]); + } + groupKeys.sort(bySeverityMessageCount); + for (var idx=0; idx<groupKeys.length; idx++) { + var k = groupKeys[idx][0]; + var messages = group[k]; + var w = parseInt(k); + var wcolor = SeverityColors[WarnPatternsSeverity[w]]; + var description = WarnPatternsDescription[w]; + if (description.length == 0) + description = "???"; + GlobalAnchor += 1; + result += "<table class='t1'><tr bgcolor='" + wcolor + "'><td>" + + "<button class='bt' id='" + GlobalAnchor + "_mark" + + "' onclick='expand(\\"" + GlobalAnchor + "\\");'>" + + "⊕</button> " + + description + " (" + messages.length + ")</td></tr></table>"; + result += "<div id='" + GlobalAnchor + + "' style='display:none;'><table class='t1'>"; + var c = 0; + messages.sort(compareMessages); + if (FlagPlatform == "chrome") { + for (var i=0; i<messages.length; i++) { + result += "<tr><td class='c" + c + "'>" + + addURLToLine(WarningMessages[messages[i][2]], WarningLinks[messages[i][3]]) + "</td></tr>"; + c = 1 - c; + } + } else { + for (var i=0; i<messages.length; i++) { + result += "<tr><td class='c" + c + "'>" + + addURL(WarningMessages[messages[i][2]]) + "</td></tr>"; + c = 1 - c; + } + } + result += "</table></div>"; + } + if (result.length > 0) { + return "<br><span style='background-color:" + color + "'><b>" + + header + ": " + totalMessages + + "</b></span><blockquote><table class='t1'>" + + result + "</table></blockquote>"; + + } + return ""; // empty section + } + function generateSectionsBySeverity() { + var result = ""; + var groups = groupWarningsBySeverity(); + for (s=0; s<SeverityColors.length; s++) { + result += createWarningSection(SeverityHeaders[s], SeverityColors[s], + groups[s]); + } + return result; + } + function generateSectionsByProject() { + var result = ""; + var groups = groupWarningsByProject(); + for (i=0; i<groups.length; i++) { + result += createWarningSection(ProjectNames[i], 'lightgrey', groups[i]); + } + return result; + } + function groupWarnings(generator) { + GlobalAnchor = 0; + var e = document.getElementById("warning_groups"); + e.innerHTML = generator(); + } + function groupBySeverity() { + groupWarnings(generateSectionsBySeverity); + } + function groupByProject() { + groupWarnings(generateSectionsByProject); + } +""" + + +# Emit a JavaScript const string +def emit_const_string(name, value, writer): + writer('const ' + name + ' = "' + escape_string(value) + '";') + + +# Emit a JavaScript const integer array. +def emit_const_int_array(name, array, writer): + writer('const ' + name + ' = [') + for n in array: + writer(str(n) + ',') + writer('];') + + +# Emit a JavaScript const string array. +def emit_const_string_array(name, array, writer): + writer('const ' + name + ' = [') + for s in array: + writer('"' + strip_escape_string(s) + '",') + writer('];') + + +# Emit a JavaScript const string array for HTML. +def emit_const_html_string_array(name, array, writer): + writer('const ' + name + ' = [') + for s in array: + # Not using html.escape yet, to work for both python 2 and 3, + # until all users switch to python 3. + # pylint:disable=deprecated-method + writer('"' + cgi.escape(strip_escape_string(s)) + '",') + writer('];') + + +# Emit a JavaScript const object array. +def emit_const_object_array(name, array, writer): + writer('const ' + name + ' = [') + for x in array: + writer(str(x) + ',') + writer('];') + + +def emit_js_data(writer, flags, warning_messages, warning_links, + warning_records, warn_patterns, project_names): + """Dump dynamic HTML page's static JavaScript data.""" + emit_const_string('FlagPlatform', flags.platform, writer) + emit_const_string('FlagURL', flags.url, writer) + emit_const_string('FlagSeparator', flags.separator, writer) + emit_const_string_array('SeverityColors', [s.color for s in Severity.levels], + writer) + emit_const_string_array('SeverityHeaders', + [s.header for s in Severity.levels], writer) + emit_const_string_array('SeverityColumnHeaders', + [s.column_header for s in Severity.levels], writer) + emit_const_string_array('ProjectNames', project_names, writer) + # pytype: disable=attribute-error + emit_const_int_array('WarnPatternsSeverity', + [w['severity'].value for w in warn_patterns], writer) + # pytype: enable=attribute-error + emit_const_html_string_array('WarnPatternsDescription', + [w['description'] for w in warn_patterns], + writer) + emit_const_html_string_array('WarningMessages', warning_messages, writer) + emit_const_object_array('Warnings', warning_records, writer) + if flags.platform == 'chrome': + emit_const_html_string_array('WarningLinks', warning_links, writer) + + +draw_table_javascript = """ +google.charts.load('current', {'packages':['table']}); +google.charts.setOnLoadCallback(drawTable); +function drawTable() { + var data = new google.visualization.DataTable(); + data.addColumn('string', StatsHeader[0]); + for (var i=1; i<StatsHeader.length; i++) { + data.addColumn('number', StatsHeader[i]); + } + data.addRows(StatsRows); + for (var i=0; i<StatsRows.length; i++) { + for (var j=0; j<StatsHeader.length; j++) { + data.setProperty(i, j, 'style', 'border:1px solid black;'); + } + } + var table = new google.visualization.Table( + document.getElementById('stats_table')); + table.draw(data, {allowHtml: true, alternatingRowStyle: true}); +} +""" + + +def dump_html(flags, output_stream, warning_messages, warning_links, + warning_records, header_str, warn_patterns, project_names): + """Dump the flags output to output_stream.""" + writer = make_writer(output_stream) + dump_html_prologue('Warnings for ' + header_str, writer, warn_patterns, + project_names) + dump_stats(writer, warn_patterns) + writer('<br><div id="stats_table"></div><br>') + writer('\n<script>') + emit_js_data(writer, flags, warning_messages, warning_links, warning_records, + warn_patterns, project_names) + writer(scripts_for_warning_groups) + writer('</script>') + emit_buttons(writer) + # Warning messages are grouped by severities or project names. + writer('<br><div id="warning_groups"></div>') + if flags.byproject: + writer('<script>groupByProject();</script>') + else: + writer('<script>groupBySeverity();</script>') + dump_fixed(writer, warn_patterns) + dump_html_epilogue(writer) + + +def write_html(flags, project_names, warn_patterns, html_path, warning_messages, + warning_links, warning_records, header_str): + """Write warnings html file.""" + if html_path: + with open(html_path, 'w') as f: + dump_html(flags, f, warning_messages, warning_links, warning_records, + header_str, warn_patterns, project_names) + + +def write_out_csv(flags, warn_patterns, warning_messages, warning_links, + warning_records, header_str, project_names): + """Write warnings csv file.""" + if flags.csvpath: + with open(flags.csvpath, 'w') as f: + dump_csv(csv.writer(f, lineterminator='\n'), warn_patterns) + + if flags.gencsv: + dump_csv(csv.writer(sys.stdout, lineterminator='\n'), warn_patterns) + else: + dump_html(flags, sys.stdout, warning_messages, warning_links, + warning_records, header_str, warn_patterns, project_names) diff --git a/tools/warn/java_warn_patterns.py b/tools/warn/java_warn_patterns.py index b5b4d9a49a..17e3864daf 100644 --- a/tools/warn/java_warn_patterns.py +++ b/tools/warn/java_warn_patterns.py @@ -16,8 +16,8 @@ """Warning patterns for Java compiler tools.""" # pylint:disable=relative-beyond-top-level -from .cpp_warn_patterns import compile_patterns # pylint:disable=g-importing-member +from .cpp_warn_patterns import compile_patterns from .severity import Severity diff --git a/tools/warn/make_warn_patterns.py b/tools/warn/make_warn_patterns.py index dd6a1b0419..4b20493ba1 100644 --- a/tools/warn/make_warn_patterns.py +++ b/tools/warn/make_warn_patterns.py @@ -16,8 +16,8 @@ """Warning patterns for build make tools.""" # pylint:disable=relative-beyond-top-level -from .cpp_warn_patterns import compile_patterns # pylint:disable=g-importing-member +from .cpp_warn_patterns import compile_patterns from .severity import Severity warn_patterns = [ diff --git a/tools/warn/other_warn_patterns.py b/tools/warn/other_warn_patterns.py index cd55aaaf92..318c3d47f5 100644 --- a/tools/warn/other_warn_patterns.py +++ b/tools/warn/other_warn_patterns.py @@ -16,8 +16,8 @@ """Warning patterns from other tools.""" # pylint:disable=relative-beyond-top-level -from .cpp_warn_patterns import compile_patterns # pylint:disable=g-importing-member +from .cpp_warn_patterns import compile_patterns from .severity import Severity diff --git a/tools/warn/tidy_warn_patterns.py b/tools/warn/tidy_warn_patterns.py index 036e2bbfd6..5416cb23d7 100644 --- a/tools/warn/tidy_warn_patterns.py +++ b/tools/warn/tidy_warn_patterns.py @@ -16,8 +16,8 @@ """Warning patterns for clang-tidy.""" # pylint:disable=relative-beyond-top-level -from .cpp_warn_patterns import compile_patterns # pylint:disable=g-importing-member +from .cpp_warn_patterns import compile_patterns from .severity import Severity diff --git a/tools/warn/warn_common.py b/tools/warn/warn_common.py index 329f1e5524..68ed995c00 100755 --- a/tools/warn/warn_common.py +++ b/tools/warn/warn_common.py @@ -45,46 +45,7 @@ Default input file is build.log, which can be changed with the --log flag. # idx to warning_links] # parse_input_file # -# To emit html page of warning messages: -# flags: --byproject, --url, --separator -# Old stuff for static html components: -# html_script_style: static html scripts and styles -# htmlbig: -# dump_stats, dump_html_prologue, dump_html_epilogue: -# emit_buttons: -# dump_fixed -# sort_warnings: -# emit_stats_by_project: -# all_patterns, -# findproject, classify_warning -# dump_html -# -# New dynamic HTML page's static JavaScript data: -# Some data are copied from Python to JavaScript, to generate HTML elements. -# FlagPlatform flags.platform -# FlagURL flags.url, used by 'android' -# FlagSeparator flags.separator, used by 'android' -# SeverityColors: list of colors for all severity levels -# SeverityHeaders: list of headers for all severity levels -# SeverityColumnHeaders: list of column_headers for all severity levels -# ProjectNames: project_names, or project_list[*][0] -# WarnPatternsSeverity: warn_patterns[*]['severity'] -# WarnPatternsDescription: warn_patterns[*]['description'] -# WarningMessages: warning_messages -# Warnings: warning_records -# StatsHeader: warning count table header row -# StatsRows: array of warning count table rows -# -# New dynamic HTML page's dynamic JavaScript data: -# -# New dynamic HTML related function to emit data: -# escape_string, strip_escape_string, emit_warning_arrays -# emit_js_data(): - -from __future__ import print_function import argparse -import cgi -import csv import io import multiprocessing import os @@ -92,15 +53,15 @@ import re import sys # pylint:disable=relative-beyond-top-level +# pylint:disable=g-importing-member from . import android_project_list from . import chrome_project_list from . import cpp_warn_patterns as cpp_patterns +from . import html_writer from . import java_warn_patterns as java_patterns from . import make_warn_patterns as make_patterns from . import other_warn_patterns as other_patterns from . import tidy_warn_patterns as tidy_patterns -# pylint:disable=g-importing-member -from .severity import Severity def parse_args(use_google3): @@ -150,304 +111,6 @@ def get_project_names(project_list): return [p[0] for p in project_list] -html_head_scripts = """\ - <script type="text/javascript"> - function expand(id) { - var e = document.getElementById(id); - var f = document.getElementById(id + "_mark"); - if (e.style.display == 'block') { - e.style.display = 'none'; - f.innerHTML = '⊕'; - } - else { - e.style.display = 'block'; - f.innerHTML = '⊖'; - } - }; - function expandCollapse(show) { - for (var id = 1; ; id++) { - var e = document.getElementById(id + ""); - var f = document.getElementById(id + "_mark"); - if (!e || !f) break; - e.style.display = (show ? 'block' : 'none'); - f.innerHTML = (show ? '⊖' : '⊕'); - } - }; - </script> - <style type="text/css"> - th,td{border-collapse:collapse; border:1px solid black;} - .button{color:blue;font-size:110%;font-weight:bolder;} - .bt{color:black;background-color:transparent;border:none;outline:none; - font-size:140%;font-weight:bolder;} - .c0{background-color:#e0e0e0;} - .c1{background-color:#d0d0d0;} - .t1{border-collapse:collapse; width:100%; border:1px solid black;} - </style> - <script src="https://www.gstatic.com/charts/loader.js"></script> -""" - - -def make_writer(output_stream): - - def writer(text): - return output_stream.write(text + '\n') - - return writer - - -def html_big(param): - return '<font size="+2">' + param + '</font>' - - -def dump_html_prologue(title, writer, warn_patterns, project_names): - writer('<html>\n<head>') - writer('<title>' + title + '</title>') - writer(html_head_scripts) - emit_stats_by_project(writer, warn_patterns, project_names) - writer('</head>\n<body>') - writer(html_big(title)) - writer('<p>') - - -def dump_html_epilogue(writer): - writer('</body>\n</head>\n</html>') - - -def sort_warnings(warn_patterns): - for i in warn_patterns: - i['members'] = sorted(set(i['members'])) - - -def create_warnings(warn_patterns, project_names): - """Creates warnings s.t. - - warnings[p][s] is as specified in above docs. - - Args: - warn_patterns: list of warning patterns for specified platform - project_names: list of project names - - Returns: - 2D warnings array where warnings[p][s] is # of warnings in project name p of - severity level s - """ - # pylint:disable=g-complex-comprehension - warnings = {p: {s.value: 0 for s in Severity.levels} for p in project_names} - for i in warn_patterns: - s = i['severity'].value - for p in i['projects']: - warnings[p][s] += i['projects'][p] - return warnings - - -def get_total_by_project(warnings, project_names): - """Returns dict, project as key and # warnings for that project as value.""" - # pylint:disable=g-complex-comprehension - return { - p: sum(warnings[p][s.value] for s in Severity.levels) - for p in project_names - } - - -def get_total_by_severity(warnings, project_names): - """Returns dict, severity as key and # warnings of that severity as value.""" - # pylint:disable=g-complex-comprehension - return { - s.value: sum(warnings[p][s.value] for p in project_names) - for s in Severity.levels - } - - -def emit_table_header(total_by_severity): - """Returns list of HTML-formatted content for severity stats.""" - - stats_header = ['Project'] - for s in Severity.levels: - if total_by_severity[s.value]: - stats_header.append( - '<span style=\'background-color:{}\'>{}</span>'.format( - s.color, s.column_header)) - stats_header.append('TOTAL') - return stats_header - - -def emit_row_counts_per_project(warnings, total_by_project, total_by_severity, - project_names): - """Returns total project warnings and row of stats for each project. - - Args: - warnings: output of create_warnings(warn_patterns, project_names) - total_by_project: output of get_total_by_project(project_names) - total_by_severity: output of get_total_by_severity(project_names) - project_names: list of project names - - Returns: - total_all_projects, the total number of warnings over all projects - stats_rows, a 2d list where each row is [Project Name, <severity counts>, - total # warnings for this project] - """ - - total_all_projects = 0 - stats_rows = [] - for p in project_names: - if total_by_project[p]: - one_row = [p] - for s in Severity.levels: - if total_by_severity[s.value]: - one_row.append(warnings[p][s.value]) - one_row.append(total_by_project[p]) - stats_rows.append(one_row) - total_all_projects += total_by_project[p] - return total_all_projects, stats_rows - - -def emit_row_counts_per_severity(total_by_severity, stats_header, stats_rows, - total_all_projects, writer): - """Emits stats_header and stats_rows as specified above. - - Args: - total_by_severity: output of get_total_by_severity() - stats_header: output of emit_table_header() - stats_rows: output of emit_row_counts_per_project() - total_all_projects: output of emit_row_counts_per_project() - writer: writer returned by make_writer(output_stream) - """ - - total_all_severities = 0 - one_row = ['<b>TOTAL</b>'] - for s in Severity.levels: - if total_by_severity[s.value]: - one_row.append(total_by_severity[s.value]) - total_all_severities += total_by_severity[s.value] - one_row.append(total_all_projects) - stats_rows.append(one_row) - writer('<script>') - emit_const_string_array('StatsHeader', stats_header, writer) - emit_const_object_array('StatsRows', stats_rows, writer) - writer(draw_table_javascript) - writer('</script>') - - -def emit_stats_by_project(writer, warn_patterns, project_names): - """Dump a google chart table of warnings per project and severity.""" - - warnings = create_warnings(warn_patterns, project_names) - total_by_project = get_total_by_project(warnings, project_names) - total_by_severity = get_total_by_severity(warnings, project_names) - stats_header = emit_table_header(total_by_severity) - total_all_projects, stats_rows = \ - emit_row_counts_per_project(warnings, total_by_project, total_by_severity, project_names) - emit_row_counts_per_severity(total_by_severity, stats_header, stats_rows, - total_all_projects, writer) - - -def dump_stats(writer, warn_patterns): - """Dump some stats about total number of warnings and such.""" - - known = 0 - skipped = 0 - unknown = 0 - sort_warnings(warn_patterns) - for i in warn_patterns: - if i['severity'] == Severity.UNMATCHED: - unknown += len(i['members']) - elif i['severity'] == Severity.SKIP: - skipped += len(i['members']) - else: - known += len(i['members']) - writer('Number of classified warnings: <b>' + str(known) + '</b><br>') - writer('Number of skipped warnings: <b>' + str(skipped) + '</b><br>') - writer('Number of unclassified warnings: <b>' + str(unknown) + '</b><br>') - total = unknown + known + skipped - extra_msg = '' - if total < 1000: - extra_msg = ' (low count may indicate incremental build)' - writer('Total number of warnings: <b>' + str(total) + '</b>' + extra_msg) - - -# New base table of warnings, [severity, warn_id, project, warning_message] -# Need buttons to show warnings in different grouping options. -# (1) Current, group by severity, id for each warning pattern -# sort by severity, warn_id, warning_message -# (2) Current --byproject, group by severity, -# id for each warning pattern + project name -# sort by severity, warn_id, project, warning_message -# (3) New, group by project + severity, -# id for each warning pattern -# sort by project, severity, warn_id, warning_message -def emit_buttons(writer): - writer('<button class="button" onclick="expandCollapse(1);">' - 'Expand all warnings</button>\n' - '<button class="button" onclick="expandCollapse(0);">' - 'Collapse all warnings</button>\n' - '<button class="button" onclick="groupBySeverity();">' - 'Group warnings by severity</button>\n' - '<button class="button" onclick="groupByProject();">' - 'Group warnings by project</button><br>') - - -def all_patterns(category): - patterns = '' - for i in category['patterns']: - patterns += i - patterns += ' / ' - return patterns - - -def dump_fixed(writer, warn_patterns): - """Show which warnings no longer occur.""" - anchor = 'fixed_warnings' - mark = anchor + '_mark' - writer('\n<br><p style="background-color:lightblue"><b>' - '<button id="' + mark + '" ' - 'class="bt" onclick="expand(\'' + anchor + '\');">' - '⊕</button> Fixed warnings. ' - 'No more occurrences. Please consider turning these into ' - 'errors if possible, before they are reintroduced in to the build' - ':</b></p>') - writer('<blockquote>') - fixed_patterns = [] - for i in warn_patterns: - if not i['members']: - fixed_patterns.append(i['description'] + ' (' + all_patterns(i) + ')') - fixed_patterns = sorted(fixed_patterns) - writer('<div id="' + anchor + '" style="display:none;"><table>') - cur_row_class = 0 - for text in fixed_patterns: - cur_row_class = 1 - cur_row_class - # remove last '\n' - t = text[:-1] if text[-1] == '\n' else text - writer('<tr><td class="c' + str(cur_row_class) + '">' + t + '</td></tr>') - writer('</table></div>') - writer('</blockquote>') - - -def write_severity(csvwriter, sev, kind, warn_patterns): - """Count warnings of given severity and write CSV entries to writer.""" - total = 0 - for pattern in warn_patterns: - if pattern['severity'] == sev and pattern['members']: - n = len(pattern['members']) - total += n - warning = kind + ': ' + (pattern['description'] or '?') - csvwriter.writerow([n, '', warning]) - # print number of warnings for each project, ordered by project name - projects = sorted(pattern['projects'].keys()) - for project in projects: - csvwriter.writerow([pattern['projects'][project], project, warning]) - csvwriter.writerow([total, '', kind + ' warnings']) - return total - - -def dump_csv(csvwriter, warn_patterns): - """Dump number of warnings in CSV format to writer.""" - sort_warnings(warn_patterns) - total = 0 - for s in Severity.levels: - total += write_severity(csvwriter, s, s.column_header, warn_patterns) - csvwriter.writerow([total, '', 'All warnings']) - - def find_project_index(line, project_patterns): for i, p in enumerate(project_patterns): if p.match(line): @@ -766,297 +429,6 @@ def parse_input_file(infile, flags): flags.platform) -# Return s with escaped backslash and quotation characters. -def escape_string(s): - return s.replace('\\', '\\\\').replace('"', '\\"') - - -# Return s without trailing '\n' and escape the quotation characters. -def strip_escape_string(s): - if not s: - return s - s = s[:-1] if s[-1] == '\n' else s - return escape_string(s) - - -def emit_warning_array(name, writer, warn_patterns): - writer('var warning_{} = ['.format(name)) - for w in warn_patterns: - if name == 'severity': - writer('{},'.format(w[name].value)) - else: - writer('{},'.format(w[name])) - writer('];') - - -def emit_warning_arrays(writer, warn_patterns): - emit_warning_array('severity', writer, warn_patterns) - writer('var warning_description = [') - for w in warn_patterns: - if w['members']: - writer('"{}",'.format(escape_string(w['description']))) - else: - writer('"",') # no such warning - writer('];') - - -scripts_for_warning_groups = """ - function compareMessages(x1, x2) { // of the same warning type - return (WarningMessages[x1[2]] <= WarningMessages[x2[2]]) ? -1 : 1; - } - function byMessageCount(x1, x2) { - return x2[2] - x1[2]; // reversed order - } - function bySeverityMessageCount(x1, x2) { - // orer by severity first - if (x1[1] != x2[1]) - return x1[1] - x2[1]; - return byMessageCount(x1, x2); - } - const ParseLinePattern = /^([^ :]+):(\\d+):(.+)/; - function addURL(line) { // used by Android - if (FlagURL == "") return line; - if (FlagSeparator == "") { - return line.replace(ParseLinePattern, - "<a target='_blank' href='" + FlagURL + "/$1'>$1</a>:$2:$3"); - } - return line.replace(ParseLinePattern, - "<a target='_blank' href='" + FlagURL + "/$1" + FlagSeparator + - "$2'>$1:$2</a>:$3"); - } - function addURLToLine(line, link) { // used by Chrome - let line_split = line.split(":"); - let path = line_split.slice(0,3).join(":"); - let msg = line_split.slice(3).join(":"); - let html_link = `<a target="_blank" href="${link}">${path}</a>${msg}`; - return html_link; - } - function createArrayOfDictionaries(n) { - var result = []; - for (var i=0; i<n; i++) result.push({}); - return result; - } - function groupWarningsBySeverity() { - // groups is an array of dictionaries, - // each dictionary maps from warning type to array of warning messages. - var groups = createArrayOfDictionaries(SeverityColors.length); - for (var i=0; i<Warnings.length; i++) { - var w = Warnings[i][0]; - var s = WarnPatternsSeverity[w]; - var k = w.toString(); - if (!(k in groups[s])) - groups[s][k] = []; - groups[s][k].push(Warnings[i]); - } - return groups; - } - function groupWarningsByProject() { - var groups = createArrayOfDictionaries(ProjectNames.length); - for (var i=0; i<Warnings.length; i++) { - var w = Warnings[i][0]; - var p = Warnings[i][1]; - var k = w.toString(); - if (!(k in groups[p])) - groups[p][k] = []; - groups[p][k].push(Warnings[i]); - } - return groups; - } - var GlobalAnchor = 0; - function createWarningSection(header, color, group) { - var result = ""; - var groupKeys = []; - var totalMessages = 0; - for (var k in group) { - totalMessages += group[k].length; - groupKeys.push([k, WarnPatternsSeverity[parseInt(k)], group[k].length]); - } - groupKeys.sort(bySeverityMessageCount); - for (var idx=0; idx<groupKeys.length; idx++) { - var k = groupKeys[idx][0]; - var messages = group[k]; - var w = parseInt(k); - var wcolor = SeverityColors[WarnPatternsSeverity[w]]; - var description = WarnPatternsDescription[w]; - if (description.length == 0) - description = "???"; - GlobalAnchor += 1; - result += "<table class='t1'><tr bgcolor='" + wcolor + "'><td>" + - "<button class='bt' id='" + GlobalAnchor + "_mark" + - "' onclick='expand(\\"" + GlobalAnchor + "\\");'>" + - "⊕</button> " + - description + " (" + messages.length + ")</td></tr></table>"; - result += "<div id='" + GlobalAnchor + - "' style='display:none;'><table class='t1'>"; - var c = 0; - messages.sort(compareMessages); - if (FlagPlatform == "chrome") { - for (var i=0; i<messages.length; i++) { - result += "<tr><td class='c" + c + "'>" + - addURLToLine(WarningMessages[messages[i][2]], WarningLinks[messages[i][3]]) + "</td></tr>"; - c = 1 - c; - } - } else { - for (var i=0; i<messages.length; i++) { - result += "<tr><td class='c" + c + "'>" + - addURL(WarningMessages[messages[i][2]]) + "</td></tr>"; - c = 1 - c; - } - } - result += "</table></div>"; - } - if (result.length > 0) { - return "<br><span style='background-color:" + color + "'><b>" + - header + ": " + totalMessages + - "</b></span><blockquote><table class='t1'>" + - result + "</table></blockquote>"; - - } - return ""; // empty section - } - function generateSectionsBySeverity() { - var result = ""; - var groups = groupWarningsBySeverity(); - for (s=0; s<SeverityColors.length; s++) { - result += createWarningSection(SeverityHeaders[s], SeverityColors[s], - groups[s]); - } - return result; - } - function generateSectionsByProject() { - var result = ""; - var groups = groupWarningsByProject(); - for (i=0; i<groups.length; i++) { - result += createWarningSection(ProjectNames[i], 'lightgrey', groups[i]); - } - return result; - } - function groupWarnings(generator) { - GlobalAnchor = 0; - var e = document.getElementById("warning_groups"); - e.innerHTML = generator(); - } - function groupBySeverity() { - groupWarnings(generateSectionsBySeverity); - } - function groupByProject() { - groupWarnings(generateSectionsByProject); - } -""" - - -# Emit a JavaScript const string -def emit_const_string(name, value, writer): - writer('const ' + name + ' = "' + escape_string(value) + '";') - - -# Emit a JavaScript const integer array. -def emit_const_int_array(name, array, writer): - writer('const ' + name + ' = [') - for n in array: - writer(str(n) + ',') - writer('];') - - -# Emit a JavaScript const string array. -def emit_const_string_array(name, array, writer): - writer('const ' + name + ' = [') - for s in array: - writer('"' + strip_escape_string(s) + '",') - writer('];') - - -# Emit a JavaScript const string array for HTML. -def emit_const_html_string_array(name, array, writer): - writer('const ' + name + ' = [') - for s in array: - # Not using html.escape yet, to work for both python 2 and 3, - # until all users switch to python 3. - # pylint:disable=deprecated-method - writer('"' + cgi.escape(strip_escape_string(s)) + '",') - writer('];') - - -# Emit a JavaScript const object array. -def emit_const_object_array(name, array, writer): - writer('const ' + name + ' = [') - for x in array: - writer(str(x) + ',') - writer('];') - - -def emit_js_data(writer, flags, warning_messages, warning_links, - warning_records, warn_patterns, project_names): - """Dump dynamic HTML page's static JavaScript data.""" - emit_const_string('FlagPlatform', flags.platform, writer) - emit_const_string('FlagURL', flags.url, writer) - emit_const_string('FlagSeparator', flags.separator, writer) - emit_const_string_array('SeverityColors', [s.color for s in Severity.levels], - writer) - emit_const_string_array('SeverityHeaders', - [s.header for s in Severity.levels], writer) - emit_const_string_array('SeverityColumnHeaders', - [s.column_header for s in Severity.levels], writer) - emit_const_string_array('ProjectNames', project_names, writer) - # pytype: disable=attribute-error - emit_const_int_array('WarnPatternsSeverity', - [w['severity'].value for w in warn_patterns], writer) - # pytype: enable=attribute-error - emit_const_html_string_array('WarnPatternsDescription', - [w['description'] for w in warn_patterns], - writer) - emit_const_html_string_array('WarningMessages', warning_messages, writer) - emit_const_object_array('Warnings', warning_records, writer) - if flags.platform == 'chrome': - emit_const_html_string_array('WarningLinks', warning_links, writer) - - -draw_table_javascript = """ -google.charts.load('current', {'packages':['table']}); -google.charts.setOnLoadCallback(drawTable); -function drawTable() { - var data = new google.visualization.DataTable(); - data.addColumn('string', StatsHeader[0]); - for (var i=1; i<StatsHeader.length; i++) { - data.addColumn('number', StatsHeader[i]); - } - data.addRows(StatsRows); - for (var i=0; i<StatsRows.length; i++) { - for (var j=0; j<StatsHeader.length; j++) { - data.setProperty(i, j, 'style', 'border:1px solid black;'); - } - } - var table = new google.visualization.Table( - document.getElementById('stats_table')); - table.draw(data, {allowHtml: true, alternatingRowStyle: true}); -} -""" - - -def dump_html(flags, output_stream, warning_messages, warning_links, - warning_records, header_str, warn_patterns, project_names): - """Dump the flags output to output_stream.""" - writer = make_writer(output_stream) - dump_html_prologue('Warnings for ' + header_str, writer, warn_patterns, - project_names) - dump_stats(writer, warn_patterns) - writer('<br><div id="stats_table"></div><br>') - writer('\n<script>') - emit_js_data(writer, flags, warning_messages, warning_links, warning_records, - warn_patterns, project_names) - writer(scripts_for_warning_groups) - writer('</script>') - emit_buttons(writer) - # Warning messages are grouped by severities or project names. - writer('<br><div id="warning_groups"></div>') - if flags.byproject: - writer('<script>groupByProject();</script>') - else: - writer('<script>groupBySeverity();</script>') - dump_fixed(writer, warn_patterns) - dump_html_epilogue(writer) - - def parse_compiler_output(compiler_output): """Parse compiler output for relevant info.""" split_output = compiler_output.split(':', 3) # 3 = max splits @@ -1153,29 +525,6 @@ def parallel_classify_warnings(warning_data, args, project_names, return warning_messages, warning_links, warning_records -def write_html(flags, project_names, warn_patterns, html_path, warning_messages, - warning_links, warning_records, header_str): - """Write warnings html file.""" - if html_path: - with open(html_path, 'w') as f: - dump_html(flags, f, warning_messages, warning_links, warning_records, - header_str, warn_patterns, project_names) - - -def write_out_csv(flags, warn_patterns, warning_messages, warning_links, - warning_records, header_str, project_names): - """Write warnings csv file.""" - if flags.csvpath: - with open(flags.csvpath, 'w') as f: - dump_csv(csv.writer(f, lineterminator='\n'), warn_patterns) - - if flags.gencsv: - dump_csv(csv.writer(sys.stdout, lineterminator='\n'), warn_patterns) - else: - dump_html(flags, sys.stdout, warning_messages, warning_links, - warning_records, header_str, warn_patterns, project_names) - - def process_log(logfile, flags, project_names, project_patterns, warn_patterns, html_path, use_google3, create_launch_subprocs_fn, classify_warnings_fn, logfile_object): @@ -1200,9 +549,9 @@ def process_log(logfile, flags, project_names, project_patterns, warn_patterns, warn_patterns, use_google3, create_launch_subprocs_fn, classify_warnings_fn) - write_html(flags, project_names, warn_patterns, html_path, - warning_messages, warning_links, warning_records, - header_str) + html_writer.write_html(flags, project_names, warn_patterns, html_path, + warning_messages, warning_links, warning_records, + header_str) return warning_messages, warning_links, warning_records, header_str @@ -1226,8 +575,9 @@ def common_main(use_google3, create_launch_subprocs_fn, classify_warnings_fn, classify_warnings_fn=classify_warnings_fn, logfile_object=logfile_object) - write_out_csv(flags, warn_patterns, warning_messages, warning_links, - warning_records, header_str, project_names) + html_writer.write_out_csv(flags, warn_patterns, warning_messages, + warning_links, warning_records, header_str, + project_names) # Return these values, so that caller can use them, if desired. return flags, warning_messages, warning_records, warn_patterns |