Use same warn_common.py and warn.py for Android and Chrome
* add chrome_project_list.py
* use newer Chrome's warn_common.py and severity.py
* modify warn_common.py to work with both python2 and python3,
to work with and without google3
Test: warn.py --url=http://cs/android --separator='?l=' build.log > warnings.html
Test: warn.py --gencsv build.log > warnings.csv
Change-Id: I9aeee7031c11988d8ead1733ff4846ae6cb1ee73
diff --git a/tools/warn/chrome_project_list.py b/tools/warn/chrome_project_list.py
new file mode 100644
index 0000000..6096522
--- /dev/null
+++ b/tools/warn/chrome_project_list.py
@@ -0,0 +1,686 @@
+# python3
+"""Clang_Tidy_Warn Project List data for Chrome.
+
+This file stores the Chrome project_list used in warn.py and
+its dependencies. It has been put into this file for easier navigation and
+unification of the Chrome and Android warn.py.
+"""
+
+
+def create_pattern(pattern):
+ return [pattern, '(^|.*/)' + pattern + '/.*: warning:']
+
+
+# A list of [project_name, file_path_pattern].
+project_list = [
+ create_pattern('android_webview'),
+ create_pattern('apps'),
+ create_pattern('ash/app_list'),
+ create_pattern('ash/public'),
+ create_pattern('ash/assistant'),
+ create_pattern('ash/display'),
+ create_pattern('ash/resources'),
+ create_pattern('ash/login'),
+ create_pattern('ash/system'),
+ create_pattern('ash/wm'),
+ create_pattern('ash/shelf'),
+ create_pattern('ash'),
+ create_pattern('base/trace_event'),
+ create_pattern('base/debug'),
+ create_pattern('base/third_party'),
+ create_pattern('base/files'),
+ create_pattern('base/test'),
+ create_pattern('base/util'),
+ create_pattern('base/task'),
+ create_pattern('base/metrics'),
+ create_pattern('base/strings'),
+ create_pattern('base/memory'),
+ create_pattern('base'),
+ create_pattern('build'),
+ create_pattern('build_overrides'),
+ create_pattern('buildtools'),
+ create_pattern('cc'),
+ create_pattern('chrome/services'),
+ create_pattern('chrome/app'),
+ create_pattern('chrome/renderer'),
+ create_pattern('chrome/test'),
+ create_pattern('chrome/common/safe_browsing'),
+ create_pattern('chrome/common/importer'),
+ create_pattern('chrome/common/media_router'),
+ create_pattern('chrome/common/extensions'),
+ create_pattern('chrome/common'),
+ create_pattern('chrome/browser/sync_file_system'),
+ create_pattern('chrome/browser/safe_browsing'),
+ create_pattern('chrome/browser/download'),
+ create_pattern('chrome/browser/ui'),
+ create_pattern('chrome/browser/supervised_user'),
+ create_pattern('chrome/browser/search'),
+ create_pattern('chrome/browser/browsing_data'),
+ create_pattern('chrome/browser/predictors'),
+ create_pattern('chrome/browser/net'),
+ create_pattern('chrome/browser/devtools'),
+ create_pattern('chrome/browser/resource_coordinator'),
+ create_pattern('chrome/browser/page_load_metrics'),
+ create_pattern('chrome/browser/extensions'),
+ create_pattern('chrome/browser/ssl'),
+ create_pattern('chrome/browser/printing'),
+ create_pattern('chrome/browser/profiles'),
+ create_pattern('chrome/browser/chromeos'),
+ create_pattern('chrome/browser/performance_manager'),
+ create_pattern('chrome/browser/metrics'),
+ create_pattern('chrome/browser/component_updater'),
+ create_pattern('chrome/browser/media'),
+ create_pattern('chrome/browser/notifications'),
+ create_pattern('chrome/browser/web_applications'),
+ create_pattern('chrome/browser/media_galleries'),
+ create_pattern('chrome/browser'),
+ create_pattern('chrome'),
+ create_pattern('chromecast'),
+ create_pattern('chromeos/services'),
+ create_pattern('chromeos/dbus'),
+ create_pattern('chromeos/assistant'),
+ create_pattern('chromeos/components'),
+ create_pattern('chromeos/settings'),
+ create_pattern('chromeos/constants'),
+ create_pattern('chromeos/network'),
+ create_pattern('chromeos'),
+ create_pattern('cloud_print'),
+ create_pattern('components/crash'),
+ create_pattern('components/subresource_filter'),
+ create_pattern('components/invalidation'),
+ create_pattern('components/autofill'),
+ create_pattern('components/onc'),
+ create_pattern('components/arc'),
+ create_pattern('components/safe_browsing'),
+ create_pattern('components/services'),
+ create_pattern('components/cast_channel'),
+ create_pattern('components/download'),
+ create_pattern('components/feed'),
+ create_pattern('components/offline_pages'),
+ create_pattern('components/bookmarks'),
+ create_pattern('components/cloud_devices'),
+ create_pattern('components/mirroring'),
+ create_pattern('components/spellcheck'),
+ create_pattern('components/viz'),
+ create_pattern('components/gcm_driver'),
+ create_pattern('components/ntp_snippets'),
+ create_pattern('components/translate'),
+ create_pattern('components/search_engines'),
+ create_pattern('components/background_task_scheduler'),
+ create_pattern('components/signin'),
+ create_pattern('components/chromeos_camera'),
+ create_pattern('components/reading_list'),
+ create_pattern('components/assist_ranker'),
+ create_pattern('components/payments'),
+ create_pattern('components/feedback'),
+ create_pattern('components/ui_devtools'),
+ create_pattern('components/password_manager'),
+ create_pattern('components/omnibox'),
+ create_pattern('components/content_settings'),
+ create_pattern('components/dom_distiller'),
+ create_pattern('components/nacl'),
+ create_pattern('components/metrics'),
+ create_pattern('components/policy'),
+ create_pattern('components/optimization_guide'),
+ create_pattern('components/exo'),
+ create_pattern('components/update_client'),
+ create_pattern('components/data_reduction_proxy'),
+ create_pattern('components/sync'),
+ create_pattern('components/drive'),
+ create_pattern('components/variations'),
+ create_pattern('components/history'),
+ create_pattern('components/webcrypto'),
+ create_pattern('components'),
+ create_pattern('content/public'),
+ create_pattern('content/renderer'),
+ create_pattern('content/test'),
+ create_pattern('content/common'),
+ create_pattern('content/browser'),
+ create_pattern('content/zygote'),
+ create_pattern('content'),
+ create_pattern('courgette'),
+ create_pattern('crypto'),
+ create_pattern('dbus'),
+ create_pattern('device/base'),
+ create_pattern('device/vr'),
+ create_pattern('device/gamepad'),
+ create_pattern('device/test'),
+ create_pattern('device/fido'),
+ create_pattern('device/bluetooth'),
+ create_pattern('device'),
+ create_pattern('docs'),
+ create_pattern('extensions/docs'),
+ create_pattern('extensions/components'),
+ create_pattern('extensions/buildflags'),
+ create_pattern('extensions/renderer'),
+ create_pattern('extensions/test'),
+ create_pattern('extensions/common'),
+ create_pattern('extensions/shell'),
+ create_pattern('extensions/browser'),
+ create_pattern('extensions/strings'),
+ create_pattern('extensions'),
+ create_pattern('fuchsia'),
+ create_pattern('gin'),
+ create_pattern('google_apis'),
+ create_pattern('google_update'),
+ create_pattern('gpu/perftests'),
+ create_pattern('gpu/GLES2'),
+ create_pattern('gpu/command_buffer'),
+ create_pattern('gpu/tools'),
+ create_pattern('gpu/gles2_conform_support'),
+ create_pattern('gpu/ipc'),
+ create_pattern('gpu/khronos_glcts_support'),
+ create_pattern('gpu'),
+ create_pattern('headless'),
+ create_pattern('infra'),
+ create_pattern('ipc'),
+ create_pattern('jingle'),
+ create_pattern('media'),
+ create_pattern('mojo'),
+ create_pattern('native_client'),
+ create_pattern('ative_client_sdk'),
+ create_pattern('net'),
+ create_pattern('out'),
+ create_pattern('pdf'),
+ create_pattern('ppapi'),
+ create_pattern('printing'),
+ create_pattern('remoting'),
+ create_pattern('rlz'),
+ create_pattern('sandbox'),
+ create_pattern('services/audio'),
+ create_pattern('services/content'),
+ create_pattern('services/data_decoder'),
+ create_pattern('services/device'),
+ create_pattern('services/file'),
+ create_pattern('services/identity'),
+ create_pattern('services/image_annotation'),
+ create_pattern('services/media_session'),
+ create_pattern('services/metrics'),
+ create_pattern('services/network'),
+ create_pattern('services/preferences'),
+ create_pattern('services/proxy_resolver'),
+ create_pattern('services/resource_coordinator'),
+ create_pattern('services/service_manager'),
+ create_pattern('services/shape_detection'),
+ create_pattern('services/strings'),
+ create_pattern('services/test'),
+ create_pattern('services/tracing'),
+ create_pattern('services/video_capture'),
+ create_pattern('services/viz'),
+ create_pattern('services/ws'),
+ create_pattern('services'),
+ create_pattern('skia/config'),
+ create_pattern('skia/ext'),
+ create_pattern('skia/public'),
+ create_pattern('skia/tools'),
+ create_pattern('skia'),
+ create_pattern('sql'),
+ create_pattern('storage'),
+ create_pattern('styleguide'),
+ create_pattern('testing'),
+ create_pattern('third_party/Python-Markdown'),
+ create_pattern('third_party/SPIRV-Tools'),
+ create_pattern('third_party/abseil-cpp'),
+ create_pattern('third_party/accessibility-audit'),
+ create_pattern('third_party/accessibility_test_framework'),
+ create_pattern('third_party/adobe'),
+ create_pattern('third_party/afl'),
+ create_pattern('third_party/android_build_tools'),
+ create_pattern('third_party/android_crazy_linker'),
+ create_pattern('third_party/android_data_chart'),
+ create_pattern('third_party/android_deps'),
+ create_pattern('third_party/android_media'),
+ create_pattern('third_party/android_ndk'),
+ create_pattern('third_party/android_opengl'),
+ create_pattern('third_party/android_platform'),
+ create_pattern('third_party/android_protobuf'),
+ create_pattern('third_party/android_sdk'),
+ create_pattern('third_party/android_support_test_runner'),
+ create_pattern('third_party/android_swipe_refresh'),
+ create_pattern('third_party/android_system_sdk'),
+ create_pattern('third_party/android_tools'),
+ create_pattern('third_party/angle'),
+ create_pattern('third_party/apache-mac'),
+ create_pattern('third_party/apache-portable-runtime'),
+ create_pattern('third_party/apache-win32'),
+ create_pattern('third_party/apk-patch-size-estimator'),
+ create_pattern('third_party/apple_apsl'),
+ create_pattern('third_party/arcore-android-sdk'),
+ create_pattern('third_party/ashmem'),
+ create_pattern('third_party/auto'),
+ create_pattern('third_party/axe-core'),
+ create_pattern('third_party/bazel'),
+ create_pattern('third_party/binutils'),
+ create_pattern('third_party/bison'),
+ create_pattern('third_party/blanketjs'),
+ create_pattern('third_party/blink/common'),
+ create_pattern('third_party/blink/manual_tests'),
+ create_pattern('third_party/blink/perf_tests'),
+ create_pattern('third_party/blink/public/common'),
+ create_pattern('third_party/blink/public/default_100_percent'),
+ create_pattern('third_party/blink/public/default_200_percent'),
+ create_pattern('third_party/blink/public/platform'),
+ create_pattern('third_party/blink/public/mojom/ad_tagging'),
+ create_pattern('third_party/blink/public/mojom/app_banner'),
+ create_pattern('third_party/blink/public/mojom/appcache'),
+ create_pattern('third_party/blink/public/mojom/array_buffer'),
+ create_pattern('third_party/blink/public/mojom/associated_interfaces'),
+ create_pattern('third_party/blink/public/mojom/autoplay'),
+ create_pattern('third_party/blink/public/mojom/background_fetch'),
+ create_pattern('third_party/blink/public/mojom/background_sync'),
+ create_pattern('third_party/blink/public/mojom/badging'),
+ create_pattern('third_party/blink/public/mojom/blob'),
+ create_pattern('third_party/blink/public/mojom/bluetooth'),
+ create_pattern('third_party/blink/public/mojom/broadcastchannel'),
+ create_pattern('third_party/blink/public/mojom/cache_storage'),
+ create_pattern('third_party/blink/public/mojom/choosers'),
+ create_pattern('third_party/blink/public/mojom/clipboard'),
+ create_pattern('third_party/blink/public/mojom/commit_result'),
+ create_pattern('third_party/blink/public/mojom/contacts'),
+ create_pattern('third_party/blink/public/mojom/cookie_store'),
+ create_pattern('third_party/blink/public/mojom/crash'),
+ create_pattern('third_party/blink/public/mojom/credentialmanager'),
+ create_pattern('third_party/blink/public/mojom/csp'),
+ create_pattern('third_party/blink/public/mojom/devtools'),
+ create_pattern('third_party/blink/public/mojom/document_metadata'),
+ create_pattern('third_party/blink/public/mojom/dom_storage'),
+ create_pattern('third_party/blink/public/mojom/dwrite_font_proxy'),
+ create_pattern('third_party/blink/public/mojom/feature_policy'),
+ create_pattern('third_party/blink/public/mojom/fetch'),
+ create_pattern('third_party/blink/public/mojom/file'),
+ create_pattern('third_party/blink/public/mojom/filesystem'),
+ create_pattern('third_party/blink/public/mojom/font_unique_name_lookup'),
+ create_pattern('third_party/blink/public/mojom/frame'),
+ create_pattern('third_party/blink/public/mojom/frame_sinks'),
+ create_pattern('third_party/blink/public/mojom/geolocation'),
+ create_pattern('third_party/blink/public/mojom/hyphenation'),
+ create_pattern('third_party/blink/public/mojom/idle'),
+ create_pattern('third_party/blink/public/mojom/indexeddb'),
+ create_pattern('third_party/blink/public/mojom/input'),
+ create_pattern('third_party/blink/public/mojom/insecure_input'),
+ create_pattern('third_party/blink/public/mojom/installation'),
+ create_pattern('third_party/blink/public/mojom/installedapp'),
+ create_pattern('third_party/blink/public/mojom/keyboard_lock'),
+ create_pattern('third_party/blink/public/mojom/leak_detector'),
+ create_pattern('third_party/blink/public/mojom/loader'),
+ create_pattern('third_party/blink/public/mojom/locks'),
+ create_pattern('third_party/blink/public/mojom/manifest'),
+ create_pattern('third_party/blink/public/mojom/media_controls'),
+ create_pattern('third_party/blink/public/mojom/mediasession'),
+ create_pattern('third_party/blink/public/mojom/mediastream'),
+ create_pattern('third_party/blink/public/mojom/messaging'),
+ create_pattern('third_party/blink/public/mojom/mime'),
+ create_pattern('third_party/blink/public/mojom/native_file_system'),
+ create_pattern('third_party/blink/public/mojom/net'),
+ create_pattern('third_party/blink/public/mojom/notifications'),
+ create_pattern('third_party/blink/public/mojom/oom_intervention'),
+ create_pattern('third_party/blink/public/mojom/page'),
+ create_pattern('third_party/blink/public/mojom/payments'),
+ create_pattern('third_party/blink/public/mojom/permissions'),
+ create_pattern('third_party/blink/public/mojom/picture_in_picture'),
+ create_pattern('third_party/blink/public/mojom/plugins'),
+ create_pattern('third_party/blink/public/mojom/portal'),
+ create_pattern('third_party/blink/public/mojom/presentation'),
+ create_pattern('third_party/blink/public/mojom/push_messaging'),
+ create_pattern('third_party/blink/public/mojom/quota'),
+ create_pattern('third_party/blink/public/mojom/remote_objects'),
+ create_pattern('third_party/blink/public/mojom/reporting'),
+ create_pattern('third_party/blink/public/mojom/script'),
+ create_pattern('third_party/blink/public/mojom/selection_menu'),
+ create_pattern('third_party/blink/public/mojom/serial'),
+ create_pattern('third_party/blink/public/mojom/service_worker'),
+ create_pattern('third_party/blink/public/mojom/site_engagement'),
+ create_pattern('third_party/blink/public/mojom/sms'),
+ create_pattern('third_party/blink/public/mojom/speech'),
+ create_pattern('third_party/blink/public/mojom/ukm'),
+ create_pattern('third_party/blink/public/mojom/unhandled_tap_notifier'),
+ create_pattern('third_party/blink/public/mojom/usb'),
+ create_pattern('third_party/blink/public/mojom/use_counter'),
+ create_pattern('third_party/blink/public/mojom/user_agent'),
+ create_pattern('third_party/blink/public/mojom/wake_lock'),
+ create_pattern('third_party/blink/public/mojom/web_client_hints'),
+ create_pattern('third_party/blink/public/mojom/web_feature'),
+ create_pattern('third_party/blink/public/mojom/webaudio'),
+ create_pattern('third_party/blink/public/mojom/webauthn'),
+ create_pattern('third_party/blink/public/mojom/webdatabase'),
+ create_pattern('third_party/blink/public/mojom/webshare'),
+ create_pattern('third_party/blink/public/mojom/window_features'),
+ create_pattern('third_party/blink/public/mojom/worker'),
+ create_pattern('third_party/blink/public/web'),
+ create_pattern('third_party/blink/renderer/bindings'),
+ create_pattern('third_party/blink/renderer/build'),
+ create_pattern('third_party/blink/renderer/controller'),
+ create_pattern('third_party/blink/renderer/core/accessibility'),
+ create_pattern('third_party/blink/renderer/core/animation'),
+ create_pattern('third_party/blink/renderer/core/aom'),
+ create_pattern('third_party/blink/renderer/core/clipboard'),
+ create_pattern('third_party/blink/renderer/core/content_capture'),
+ create_pattern('third_party/blink/renderer/core/context_features'),
+ create_pattern('third_party/blink/renderer/core/css'),
+ create_pattern('third_party/blink/renderer/core/display_lock'),
+ create_pattern('third_party/blink/renderer/core/dom'),
+ create_pattern('third_party/blink/renderer/core/editing'),
+ create_pattern('third_party/blink/renderer/core/events'),
+ create_pattern('third_party/blink/renderer/core/execution_context'),
+ create_pattern('third_party/blink/renderer/core/exported'),
+ create_pattern('third_party/blink/renderer/core/feature_policy'),
+ create_pattern('third_party/blink/renderer/core/fetch'),
+ create_pattern('third_party/blink/renderer/core/fileapi'),
+ create_pattern('third_party/blink/renderer/core/frame'),
+ create_pattern('third_party/blink/renderer/core/fullscreen'),
+ create_pattern('third_party/blink/renderer/core/geometry'),
+ create_pattern('third_party/blink/renderer/core/html'),
+ create_pattern('third_party/blink/renderer/core/imagebitmap'),
+ create_pattern('third_party/blink/renderer/core/input'),
+ create_pattern('third_party/blink/renderer/core/inspector'),
+ create_pattern('third_party/blink/renderer/core/intersection_observer'),
+ create_pattern('third_party/blink/renderer/core/invisible_dom'),
+ create_pattern('third_party/blink/renderer/core/layout'),
+ create_pattern('third_party/blink/renderer/core/loader'),
+ create_pattern('third_party/blink/renderer/core/messaging'),
+ create_pattern('third_party/blink/renderer/core/mojo'),
+ create_pattern('third_party/blink/renderer/core/offscreencanvas'),
+ create_pattern('third_party/blink/renderer/core/origin_trials'),
+ create_pattern('third_party/blink/renderer/core/page'),
+ create_pattern('third_party/blink/renderer/core/paint'),
+ create_pattern('third_party/blink/renderer/core/probe'),
+ create_pattern('third_party/blink/renderer/core/resize_observer'),
+ create_pattern('third_party/blink/renderer/core/scheduler'),
+ create_pattern('third_party/blink/renderer/core/script'),
+ create_pattern('third_party/blink/renderer/core/scroll'),
+ create_pattern('third_party/blink/renderer/core/streams'),
+ create_pattern('third_party/blink/renderer/core/style'),
+ create_pattern('third_party/blink/renderer/core/svg'),
+ create_pattern('third_party/blink/renderer/core/testing'),
+ create_pattern('third_party/blink/renderer/core/timezone'),
+ create_pattern('third_party/blink/renderer/core/timing'),
+ create_pattern('third_party/blink/renderer/core/trustedtypes'),
+ create_pattern('third_party/blink/renderer/core/typed_arrays'),
+ create_pattern('third_party/blink/renderer/core/url'),
+ create_pattern('third_party/blink/renderer/core/win'),
+ create_pattern('third_party/blink/renderer/core/workers'),
+ create_pattern('third_party/blink/renderer/core/xml'),
+ create_pattern('third_party/blink/renderer/core/xmlhttprequest'),
+ create_pattern('third_party/blink/renderer/devtools'),
+ create_pattern('third_party/blink/renderer/modules'),
+ create_pattern('third_party/blink/renderer/platform'),
+ create_pattern('third_party/blink/tools'),
+ create_pattern('third_party/blink/web_tests'),
+ create_pattern('third_party/boringssl'),
+ create_pattern('third_party/bouncycastle'),
+ create_pattern('third_party/breakpad'),
+ create_pattern('third_party/brotli'),
+ create_pattern('third_party/bspatch'),
+ create_pattern('third_party/byte_buddy'),
+ create_pattern('third_party/cacheinvalidation'),
+ create_pattern('third_party/catapult'),
+ create_pattern('third_party/cct_dynamic_module'),
+ create_pattern('third_party/ced'),
+ create_pattern('third_party/chaijs'),
+ create_pattern('third_party/checkstyle'),
+ create_pattern('third_party/chromevox'),
+ create_pattern('third_party/chromite'),
+ create_pattern('third_party/cld_3'),
+ create_pattern('third_party/closure_compiler'),
+ create_pattern('third_party/colorama'),
+ create_pattern('third_party/crashpad'),
+ create_pattern('third_party/crc32c'),
+ create_pattern('third_party/cros_system_api'),
+ create_pattern('third_party/custom_tabs_client'),
+ create_pattern('third_party/d3'),
+ create_pattern('third_party/dav1d'),
+ create_pattern('third_party/dawn'),
+ create_pattern('third_party/decklink'),
+ create_pattern('third_party/depot_tools'),
+ create_pattern('third_party/devscripts'),
+ create_pattern('third_party/devtools-node-modules'),
+ create_pattern('third_party/dom_distiller_js'),
+ create_pattern('third_party/elfutils'),
+ create_pattern('third_party/emoji-segmenter'),
+ create_pattern('third_party/errorprone'),
+ create_pattern('third_party/espresso'),
+ create_pattern('third_party/expat'),
+ create_pattern('third_party/feed'),
+ create_pattern('third_party/ffmpeg'),
+ create_pattern('third_party/flac'),
+ create_pattern('third_party/flatbuffers'),
+ create_pattern('third_party/flot'),
+ create_pattern('third_party/fontconfig'),
+ create_pattern('third_party/freetype'),
+ create_pattern('third_party/fuchsia-sdk'),
+ create_pattern('third_party/gestures'),
+ create_pattern('third_party/gif_player'),
+ create_pattern('third_party/glfw'),
+ create_pattern('third_party/glslang'),
+ create_pattern('third_party/gnu_binutils'),
+ create_pattern('third_party/google-truth'),
+ create_pattern('third_party/google_android_play_core'),
+ create_pattern('third_party/google_appengine_cloudstorage'),
+ create_pattern('third_party/google_input_tools'),
+ create_pattern('third_party/google_toolbox_for_mac'),
+ create_pattern('third_party/google_trust_services'),
+ create_pattern('third_party/googletest'),
+ create_pattern('third_party/gperf'),
+ create_pattern('third_party/gradle_wrapper'),
+ create_pattern('third_party/grpc'),
+ create_pattern('third_party/gson'),
+ create_pattern('third_party/guava'),
+ create_pattern('third_party/gvr-android-keyboard'),
+ create_pattern('third_party/gvr-android-sdk'),
+ create_pattern('third_party/hamcrest'),
+ create_pattern('third_party/harfbuzz-ng'),
+ create_pattern('third_party/hunspell'),
+ create_pattern('third_party/hunspell_dictionaries'),
+ create_pattern('third_party/iaccessible2'),
+ create_pattern('third_party/iccjpeg'),
+ create_pattern('third_party/icu/android'),
+ create_pattern('third_party/icu/android_small'),
+ create_pattern('third_party/icu/cast'),
+ create_pattern('third_party/icu/chromeos'),
+ create_pattern('third_party/icu/common'),
+ create_pattern('third_party/icu/filters'),
+ create_pattern('third_party/icu/flutter'),
+ create_pattern('third_party/icu/fuzzers'),
+ create_pattern('third_party/icu/ios'),
+ create_pattern('third_party/icu/patches'),
+ create_pattern('third_party/icu/scripts'),
+ create_pattern('third_party/icu/source'),
+ create_pattern('third_party/icu/tzres'),
+ create_pattern('third_party/icu4j'),
+ create_pattern('third_party/ijar'),
+ create_pattern('third_party/ink'),
+ create_pattern('third_party/inspector_protocol'),
+ create_pattern('third_party/instrumented_libraries'),
+ create_pattern('third_party/intellij'),
+ create_pattern('third_party/isimpledom'),
+ create_pattern('third_party/jacoco'),
+ create_pattern('third_party/jinja2'),
+ create_pattern('third_party/jsoncpp'),
+ create_pattern('third_party/jsr-305'),
+ create_pattern('third_party/jstemplate'),
+ create_pattern('third_party/junit'),
+ create_pattern('third_party/khronos'),
+ create_pattern('third_party/lcov'),
+ create_pattern('third_party/leveldatabase'),
+ create_pattern('third_party/libFuzzer'),
+ create_pattern('third_party/libXNVCtrl'),
+ create_pattern('third_party/libaddressinput'),
+ create_pattern('third_party/libaom'),
+ create_pattern('third_party/libcxx-pretty-printers'),
+ create_pattern('third_party/libdrm'),
+ create_pattern('third_party/libevdev'),
+ create_pattern('third_party/libjingle_xmpp'),
+ create_pattern('third_party/libjpeg'),
+ create_pattern('third_party/libjpeg_turbo'),
+ create_pattern('third_party/liblouis'),
+ create_pattern('third_party/libovr'),
+ create_pattern('third_party/libphonenumber'),
+ create_pattern('third_party/libpng'),
+ create_pattern('third_party/libprotobuf-mutator'),
+ create_pattern('third_party/libsecret'),
+ create_pattern('third_party/libsrtp'),
+ create_pattern('third_party/libsync'),
+ create_pattern('third_party/libudev'),
+ create_pattern('third_party/libusb'),
+ create_pattern('third_party/libvpx'),
+ create_pattern('third_party/libwebm'),
+ create_pattern('third_party/libwebp'),
+ create_pattern('third_party/libxml'),
+ create_pattern('third_party/libxslt'),
+ create_pattern('third_party/libyuv'),
+ create_pattern('third_party/lighttpd'),
+ create_pattern('third_party/logilab'),
+ create_pattern('third_party/lss'),
+ create_pattern('third_party/lzma_sdk'),
+ create_pattern('third_party/mach_override'),
+ create_pattern('third_party/markdown'),
+ create_pattern('third_party/markupsafe'),
+ create_pattern('third_party/material_design_icons'),
+ create_pattern('third_party/mesa_headers'),
+ create_pattern('third_party/metrics_proto'),
+ create_pattern('third_party/microsoft_webauthn'),
+ create_pattern('third_party/mingw-w64'),
+ create_pattern('third_party/minigbm'),
+ create_pattern('third_party/minizip'),
+ create_pattern('third_party/mocha'),
+ create_pattern('third_party/mockito'),
+ create_pattern('third_party/modp_b64'),
+ create_pattern('third_party/motemplate'),
+ create_pattern('third_party/mozilla'),
+ create_pattern('third_party/nacl_sdk_binaries'),
+ create_pattern('third_party/nasm'),
+ create_pattern('third_party/netty-tcnative'),
+ create_pattern('third_party/netty4'),
+ create_pattern('third_party/node'),
+ create_pattern('third_party/nvml'),
+ create_pattern('third_party/objenesis'),
+ create_pattern('third_party/ocmock'),
+ create_pattern('third_party/openh264'),
+ create_pattern('third_party/openscreen'),
+ create_pattern('third_party/openvr'),
+ create_pattern('third_party/opus'),
+ create_pattern('third_party/ots'),
+ create_pattern('third_party/ow2_asm'),
+ create_pattern('third_party/pdfium'),
+ create_pattern('third_party/pefile'),
+ create_pattern('third_party/perfetto'),
+ create_pattern('third_party/perl'),
+ create_pattern('third_party/pexpect'),
+ create_pattern('third_party/pffft'),
+ create_pattern('third_party/ply'),
+ create_pattern('third_party/polymer'),
+ create_pattern('third_party/proguard'),
+ create_pattern('third_party/protobuf'),
+ create_pattern('third_party/protoc_javalite'),
+ create_pattern('third_party/pycoverage'),
+ create_pattern('third_party/pyelftools'),
+ create_pattern('third_party/pyjson5'),
+ create_pattern('third_party/pylint'),
+ create_pattern('third_party/pymock'),
+ create_pattern('third_party/pystache'),
+ create_pattern('third_party/pywebsocket'),
+ create_pattern('third_party/qcms'),
+ create_pattern('third_party/quic_trace'),
+ create_pattern('third_party/qunit'),
+ create_pattern('third_party/r8'),
+ create_pattern('third_party/re2'),
+ create_pattern('third_party/requests'),
+ create_pattern('third_party/rnnoise'),
+ create_pattern('third_party/robolectric'),
+ create_pattern('third_party/s2cellid'),
+ create_pattern('third_party/sfntly'),
+ create_pattern('third_party/shaderc'),
+ create_pattern('third_party/simplejson'),
+ create_pattern('third_party/sinonjs'),
+ create_pattern('third_party/skia'),
+ create_pattern('third_party/smhasher'),
+ create_pattern('third_party/snappy'),
+ create_pattern('third_party/speech-dispatcher'),
+ create_pattern('third_party/spirv-cross'),
+ create_pattern('third_party/spirv-headers'),
+ create_pattern('third_party/sqlite'),
+ create_pattern('third_party/sqlite4java'),
+ create_pattern('third_party/sudden_motion_sensor'),
+ create_pattern('third_party/swiftshader'),
+ create_pattern('third_party/tcmalloc'),
+ create_pattern('third_party/test_fonts'),
+ create_pattern('third_party/tlslite'),
+ create_pattern('third_party/ub-uiautomator'),
+ create_pattern('third_party/unrar'),
+ create_pattern('third_party/usb_ids'),
+ create_pattern('third_party/usrsctp'),
+ create_pattern('third_party/v4l-utils'),
+ create_pattern('third_party/vulkan'),
+ create_pattern('third_party/wayland'),
+ create_pattern('third_party/wayland-protocols'),
+ create_pattern('third_party/wds'),
+ create_pattern('third_party/web-animations-js'),
+ create_pattern('third_party/webdriver'),
+ create_pattern('third_party/webgl'),
+ create_pattern('third_party/webrtc'),
+ create_pattern('third_party/webrtc_overrides'),
+ create_pattern('third_party/webxr_test_pages'),
+ create_pattern('third_party/widevine'),
+ create_pattern('third_party/win_build_output'),
+ create_pattern('third_party/woff2'),
+ create_pattern('third_party/wtl'),
+ create_pattern('third_party/xdg-utils'),
+ create_pattern('third_party/xstream'),
+ create_pattern('third_party/yasm'),
+ create_pattern('third_party/zlib'),
+ create_pattern('tools'),
+ create_pattern('ui/accelerated_widget_mac'),
+ create_pattern('ui/accessibility'),
+ create_pattern('ui/android'),
+ create_pattern('ui/aura'),
+ create_pattern('ui/aura_extra'),
+ create_pattern('ui/base'),
+ create_pattern('ui/chromeos'),
+ create_pattern('ui/compositor'),
+ create_pattern('ui/compositor_extra'),
+ create_pattern('ui/content_accelerators'),
+ create_pattern('ui/display'),
+ create_pattern('ui/events'),
+ create_pattern('ui/file_manager'),
+ create_pattern('ui/gfx'),
+ create_pattern('ui/gl'),
+ create_pattern('ui/latency'),
+ create_pattern('ui/login'),
+ create_pattern('ui/message_center'),
+ create_pattern('ui/native_theme'),
+ create_pattern('ui/ozone'),
+ create_pattern('ui/platform_window'),
+ create_pattern('ui/resources'),
+ create_pattern('ui/shell_dialogs'),
+ create_pattern('ui/snapshot'),
+ create_pattern('ui/strings'),
+ create_pattern('ui/surface'),
+ create_pattern('ui/touch_selection'),
+ create_pattern('ui/views'),
+ create_pattern('ui/views_bridge_mac'),
+ create_pattern('ui/views_content_client'),
+ create_pattern('ui/web_dialogs'),
+ create_pattern('ui/webui'),
+ create_pattern('ui/wm'),
+ create_pattern('url'),
+ create_pattern('v8/benchmarks'),
+ create_pattern('v8/build_overrides'),
+ create_pattern('v8/custom_deps'),
+ create_pattern('v8/docs'),
+ create_pattern('v8/gni'),
+ create_pattern('v8/include'),
+ create_pattern('v8/infra'),
+ create_pattern('v8/samples'),
+ create_pattern('v8/src'),
+ create_pattern('v8/test'),
+ create_pattern('v8/testing'),
+ create_pattern('v8/third_party'),
+ create_pattern('v8/tools'),
+
+ # keep out/obj and other patterns at the end.
+ [
+ 'out/obj', '.*/(gen|obj[^/]*)/(include|EXECUTABLES|SHARED_LIBRARIES|'
+ 'STATIC_LIBRARIES|NATIVE_TESTS)/.*: warning:'
+ ],
+ ['other', '.*'] # all other unrecognized patterns
+]
diff --git a/tools/warn/severity.py b/tools/warn/severity.py
index b1c38e4..b4c03c9 100644
--- a/tools/warn/severity.py
+++ b/tools/warn/severity.py
@@ -20,24 +20,26 @@
# pylint:disable=old-style-class
+class SeverityInfo:
+
+ def __init__(self, value, color, column_header, header):
+ self.value = value
+ self.color = color
+ self.column_header = column_header
+ self.header = header
+
+
+# pylint:disable=old-style-class
class Severity:
"""Class of Severity levels where each level is a SeverityInfo."""
- class SeverityInfo:
-
- def __init__(self, value, color, column_header, header):
- self.value = value
- self.color = color
- self.column_header = column_header
- self.header = header
-
# SEVERITY_UNKNOWN should never occur since every warn_pattern listed has
# a specified severity. It exists for protobuf, the other values must
# map to non-zero values (since 0 is reserved for a default UNKNOWN), but
# logic in clang_tidy_warn.py assumes severity level values are consecutive
# ints starting with 0.
- SEVERITY_UNKNOWN = SeverityInfo(0, 'blueviolet', 'Errors of unknown severity',
- 'Unknown severity (should not occur)')
+ SEVERITY_UNKNOWN = SeverityInfo(0, 'blueviolet', 'Unknown',
+ 'Unknown-severity warnings)')
FIXMENOW = SeverityInfo(1, 'fuschia', 'FixNow',
'Critical warnings, fix me now')
HIGH = SeverityInfo(2, 'red', 'High', 'High severity warnings')
diff --git a/tools/warn/warn.py b/tools/warn/warn.py
index bdfd489..56e8787 100755
--- a/tools/warn/warn.py
+++ b/tools/warn/warn.py
@@ -17,21 +17,51 @@
"""Simple wrapper to run warn_common with Python standard Pool."""
import multiprocessing
+import signal
+import sys
# pylint:disable=relative-beyond-top-level
-# pylint:disable=g-importing-member
-from .warn_common import common_main
+from . import warn_common as common
-# This parallel_process could be changed depending on platform
-# and availability of multi-process library functions.
-def parallel_process(num_cpu, classify_warnings, groups):
+def classify_warnings(args):
+ """Classify a list of warning lines.
+
+ Args:
+ args: dictionary {
+ 'group': list of (warning, link),
+ 'project_patterns': re.compile(project_list[p][1]),
+ 'warn_patterns': list of warn_pattern,
+ 'num_processes': number of processes being used for multiprocessing }
+ Returns:
+ results: a list of the classified warnings.
+ """
+ results = []
+ for line, link in args['group']:
+ common.classify_one_warning(line, link, results, args['project_patterns'],
+ args['warn_patterns'])
+
+ # After the main work, ignore all other signals to a child process,
+ # to avoid bad warning/error messages from the exit clean-up process.
+ if args['num_processes'] > 1:
+ signal.signal(signal.SIGTERM, lambda *args: sys.exit(-signal.SIGTERM))
+ return results
+
+
+def create_and_launch_subprocesses(num_cpu, classify_warnings_fn, arg_groups,
+ group_results):
pool = multiprocessing.Pool(num_cpu)
- return pool.map(classify_warnings, groups)
+ for cpu in range(num_cpu):
+ proc_result = pool.map(classify_warnings_fn, arg_groups[cpu])
+ if proc_result is not None:
+ group_results.append(proc_result)
+ return group_results
def main():
- common_main(parallel_process)
+ use_google3 = False
+ common.common_main(use_google3, create_and_launch_subprocesses,
+ classify_warnings)
if __name__ == '__main__':
diff --git a/tools/warn/warn_common.py b/tools/warn/warn_common.py
index 0c9d9ef..329f1e5 100755
--- a/tools/warn/warn_common.py
+++ b/tools/warn/warn_common.py
@@ -18,6 +18,8 @@
Default is to output warnings in HTML tables grouped by warning severity.
Use option --byproject to output tables grouped by source file projects.
Use option --gencsv to output warning counts in CSV format.
+
+Default input file is build.log, which can be changed with the --log flag.
"""
# List of important data structures and functions in this script.
@@ -36,13 +38,11 @@
# project_patterns[p] re.compile(project_list[p][1])
# project_names[p] project_list[p][0]
# warning_messages array of each warning message, without source url
+# warning_links array of each warning code search link; for 'chrome'
# warning_records array of [idx to warn_patterns,
# idx to project_names,
-# idx to warning_messages]
-# android_root
-# platform_version
-# target_product
-# target_variant
+# idx to warning_messages,
+# idx to warning_links]
# parse_input_file
#
# To emit html page of warning messages:
@@ -61,8 +61,9 @@
#
# New dynamic HTML page's static JavaScript data:
# Some data are copied from Python to JavaScript, to generate HTML elements.
-# FlagURL args.url
-# FlagSeparator args.separator
+# FlagPlatform flags.platform
+# FlagURL flags.url, used by 'android'
+# FlagSeparator flags.separator, used by 'android'
# SeverityColors: list of colors for all severity levels
# SeverityHeaders: list of headers for all severity levels
# SeverityColumnHeaders: list of column_headers for all severity levels
@@ -88,80 +89,67 @@
import multiprocessing
import os
import re
-import signal
import sys
# pylint:disable=relative-beyond-top-level
-from . import cpp_warn_patterns
-from . import java_warn_patterns
-from . import make_warn_patterns
-from . import other_warn_patterns
-from . import tidy_warn_patterns
+from . import android_project_list
+from . import chrome_project_list
+from . import cpp_warn_patterns as cpp_patterns
+from . import java_warn_patterns as java_patterns
+from . import make_warn_patterns as make_patterns
+from . import other_warn_patterns as other_patterns
+from . import tidy_warn_patterns as tidy_patterns
# pylint:disable=g-importing-member
-from .android_project_list import project_list
from .severity import Severity
-parser = argparse.ArgumentParser(description='Convert a build log into HTML')
-parser.add_argument('--csvpath',
- help='Save CSV warning file to the passed absolute path',
- default=None)
-parser.add_argument('--gencsv',
- help='Generate a CSV file with number of various warnings',
- action='store_true',
- default=False)
-parser.add_argument('--byproject',
- help='Separate warnings in HTML output by project names',
- action='store_true',
- default=False)
-parser.add_argument('--url',
- help='Root URL of an Android source code tree prefixed '
- 'before files in warnings')
-parser.add_argument('--separator',
- help='Separator between the end of a URL and the line '
- 'number argument. e.g. #')
-parser.add_argument('--processes',
- type=int,
- default=multiprocessing.cpu_count(),
- help='Number of parallel processes to process warnings')
-parser.add_argument(dest='buildlog', metavar='build.log',
- help='Path to build.log file')
-args = parser.parse_args()
-warn_patterns = make_warn_patterns.warn_patterns
-warn_patterns.extend(cpp_warn_patterns.warn_patterns)
-warn_patterns.extend(java_warn_patterns.warn_patterns)
-warn_patterns.extend(tidy_warn_patterns.warn_patterns)
-warn_patterns.extend(other_warn_patterns.warn_patterns)
-
-project_patterns = []
-project_names = []
-warning_messages = []
-warning_records = []
+def parse_args(use_google3):
+ """Define and parse the args. Return the parse_args() result."""
+ parser = argparse.ArgumentParser(
+ description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ parser.add_argument('--capacitor_path', default='',
+ help='Save capacitor warning file to the passed absolute'
+ ' path')
+ # csvpath has a different naming than the above path because historically the
+ # original Android script used csvpath, so other scripts rely on it
+ parser.add_argument('--csvpath', default='',
+ help='Save CSV warning file to the passed path')
+ parser.add_argument('--gencsv', action='store_true',
+ help='Generate CSV file with number of various warnings')
+ parser.add_argument('--byproject', action='store_true',
+ help='Separate warnings in HTML output by project names')
+ parser.add_argument('--url', default='',
+ help='Root URL of an Android source code tree prefixed '
+ 'before files in warnings')
+ parser.add_argument('--separator', default='?l=',
+ help='Separator between the end of a URL and the line '
+ 'number argument. e.g. #')
+ parser.add_argument('--processes', default=multiprocessing.cpu_count(),
+ type=int,
+ help='Number of parallel processes to process warnings')
+ # Old Android build scripts call warn.py without --platform,
+ # so the default platform is set to 'android'.
+ parser.add_argument('--platform', default='android',
+ choices=['chrome', 'android'],
+ help='Platform of the build log')
+ # Old Android build scripts call warn.py with only a build.log file path.
+ parser.add_argument('--log', help='Path to build log file')
+ parser.add_argument(dest='buildlog', metavar='build.log',
+ default='build.log', nargs='?',
+ help='Path to build.log file')
+ flags = parser.parse_args()
+ if not flags.log:
+ flags.log = flags.buildlog
+ if not use_google3 and not os.path.exists(flags.log):
+ sys.exit('Cannot find log file: ' + flags.log)
+ return flags
-def initialize_arrays():
- """Complete global arrays before they are used."""
- global project_names, project_patterns
- project_names = [p[0] for p in project_list]
- project_patterns = [re.compile(p[1]) for p in project_list]
- for w in warn_patterns:
- w['members'] = []
- # Each warning pattern has a 'projects' dictionary, that
- # maps a project name to number of warnings in that project.
- w['projects'] = {}
+def get_project_names(project_list):
+ """Get project_names from project_list."""
+ return [p[0] for p in project_list]
-initialize_arrays()
-
-
-android_root = ''
-platform_version = 'unknown'
-target_product = 'unknown'
-target_variant = 'unknown'
-
-
-##### Data and functions to dump html file. ##################################
-
html_head_scripts = """\
<script type="text/javascript">
function expand(id) {
@@ -211,11 +199,11 @@
return '<font size="+2">' + param + '</font>'
-def dump_html_prologue(title, writer):
+def dump_html_prologue(title, writer, warn_patterns, project_names):
writer('<html>\n<head>')
writer('<title>' + title + '</title>')
writer(html_head_scripts)
- emit_stats_by_project(writer)
+ emit_stats_by_project(writer, warn_patterns, project_names)
writer('</head>\n<body>')
writer(html_big(title))
writer('<p>')
@@ -225,36 +213,54 @@
writer('</body>\n</head>\n</html>')
-def sort_warnings():
+def sort_warnings(warn_patterns):
for i in warn_patterns:
i['members'] = sorted(set(i['members']))
-def emit_stats_by_project(writer):
- """Dump a google chart table of warnings per project and severity."""
- # warnings[p][s] is number of warnings in project p of severity s.
+def create_warnings(warn_patterns, project_names):
+ """Creates warnings s.t.
+
+ warnings[p][s] is as specified in above docs.
+
+ Args:
+ warn_patterns: list of warning patterns for specified platform
+ project_names: list of project names
+
+ Returns:
+ 2D warnings array where warnings[p][s] is # of warnings in project name p of
+ severity level s
+ """
# pylint:disable=g-complex-comprehension
warnings = {p: {s.value: 0 for s in Severity.levels} for p in project_names}
for i in warn_patterns:
- # pytype: disable=attribute-error
s = i['severity'].value
- # pytype: enable=attribute-error
for p in i['projects']:
warnings[p][s] += i['projects'][p]
+ return warnings
- # total_by_project[p] is number of warnings in project p.
- total_by_project = {
+
+def get_total_by_project(warnings, project_names):
+ """Returns dict, project as key and # warnings for that project as value."""
+ # pylint:disable=g-complex-comprehension
+ return {
p: sum(warnings[p][s.value] for s in Severity.levels)
for p in project_names
}
- # total_by_severity[s] is number of warnings of severity s.
- total_by_severity = {
+
+def get_total_by_severity(warnings, project_names):
+ """Returns dict, severity as key and # warnings of that severity as value."""
+ # pylint:disable=g-complex-comprehension
+ return {
s.value: sum(warnings[p][s.value] for p in project_names)
for s in Severity.levels
}
- # emit table header
+
+def emit_table_header(total_by_severity):
+ """Returns list of HTML-formatted content for severity stats."""
+
stats_header = ['Project']
for s in Severity.levels:
if total_by_severity[s.value]:
@@ -262,8 +268,25 @@
'<span style=\'background-color:{}\'>{}</span>'.format(
s.color, s.column_header))
stats_header.append('TOTAL')
+ return stats_header
- # emit a row of warning counts per project, skip no-warning projects
+
+def emit_row_counts_per_project(warnings, total_by_project, total_by_severity,
+ project_names):
+ """Returns total project warnings and row of stats for each project.
+
+ Args:
+ warnings: output of create_warnings(warn_patterns, project_names)
+ total_by_project: output of get_total_by_project(project_names)
+ total_by_severity: output of get_total_by_severity(project_names)
+ project_names: list of project names
+
+ Returns:
+ total_all_projects, the total number of warnings over all projects
+ stats_rows, a 2d list where each row is [Project Name, <severity counts>,
+ total # warnings for this project]
+ """
+
total_all_projects = 0
stats_rows = []
for p in project_names:
@@ -275,8 +298,21 @@
one_row.append(total_by_project[p])
stats_rows.append(one_row)
total_all_projects += total_by_project[p]
+ return total_all_projects, stats_rows
- # emit a row of warning counts per severity
+
+def emit_row_counts_per_severity(total_by_severity, stats_header, stats_rows,
+ total_all_projects, writer):
+ """Emits stats_header and stats_rows as specified above.
+
+ Args:
+ total_by_severity: output of get_total_by_severity()
+ stats_header: output of emit_table_header()
+ stats_rows: output of emit_row_counts_per_project()
+ total_all_projects: output of emit_row_counts_per_project()
+ writer: writer returned by make_writer(output_stream)
+ """
+
total_all_severities = 0
one_row = ['<b>TOTAL</b>']
for s in Severity.levels:
@@ -292,12 +328,26 @@
writer('</script>')
-def dump_stats(writer):
+def emit_stats_by_project(writer, warn_patterns, project_names):
+ """Dump a google chart table of warnings per project and severity."""
+
+ warnings = create_warnings(warn_patterns, project_names)
+ total_by_project = get_total_by_project(warnings, project_names)
+ total_by_severity = get_total_by_severity(warnings, project_names)
+ stats_header = emit_table_header(total_by_severity)
+ total_all_projects, stats_rows = \
+ emit_row_counts_per_project(warnings, total_by_project, total_by_severity, project_names)
+ emit_row_counts_per_severity(total_by_severity, stats_header, stats_rows,
+ total_all_projects, writer)
+
+
+def dump_stats(writer, warn_patterns):
"""Dump some stats about total number of warnings and such."""
+
known = 0
skipped = 0
unknown = 0
- sort_warnings()
+ sort_warnings(warn_patterns)
for i in warn_patterns:
if i['severity'] == Severity.UNMATCHED:
unknown += len(i['members'])
@@ -344,7 +394,7 @@
return patterns
-def dump_fixed(writer):
+def dump_fixed(writer, warn_patterns):
"""Show which warnings no longer occur."""
anchor = 'fixed_warnings'
mark = anchor + '_mark'
@@ -372,95 +422,156 @@
writer('</blockquote>')
-def find_project_index(line):
- for p in range(len(project_patterns)):
- if project_patterns[p].match(line):
- return p
+def write_severity(csvwriter, sev, kind, warn_patterns):
+ """Count warnings of given severity and write CSV entries to writer."""
+ total = 0
+ for pattern in warn_patterns:
+ if pattern['severity'] == sev and pattern['members']:
+ n = len(pattern['members'])
+ total += n
+ warning = kind + ': ' + (pattern['description'] or '?')
+ csvwriter.writerow([n, '', warning])
+ # print number of warnings for each project, ordered by project name
+ projects = sorted(pattern['projects'].keys())
+ for project in projects:
+ csvwriter.writerow([pattern['projects'][project], project, warning])
+ csvwriter.writerow([total, '', kind + ' warnings'])
+ return total
+
+
+def dump_csv(csvwriter, warn_patterns):
+ """Dump number of warnings in CSV format to writer."""
+ sort_warnings(warn_patterns)
+ total = 0
+ for s in Severity.levels:
+ total += write_severity(csvwriter, s, s.column_header, warn_patterns)
+ csvwriter.writerow([total, '', 'All warnings'])
+
+
+def find_project_index(line, project_patterns):
+ for i, p in enumerate(project_patterns):
+ if p.match(line):
+ return i
return -1
-def classify_one_warning(line, results):
+def classify_one_warning(warning, link, results, project_patterns,
+ warn_patterns):
"""Classify one warning line."""
- for i in range(len(warn_patterns)):
- w = warn_patterns[i]
+ for i, w in enumerate(warn_patterns):
for cpat in w['compiled_patterns']:
- # pytype: disable=attribute-error
- if cpat.match(line):
- p = find_project_index(line)
- results.append([line, i, p])
+ if cpat.match(warning):
+ p = find_project_index(warning, project_patterns)
+ results.append([warning, link, i, p])
return
else:
# If we end up here, there was a problem parsing the log
# probably caused by 'make -j' mixing the output from
# 2 or more concurrent compiles
pass
- # pytype: enable=attribute-error
-def classify_warnings(lines):
- results = []
- for line in lines:
- classify_one_warning(line, results)
- # After the main work, ignore all other signals to a child process,
- # to avoid bad warning/error messages from the exit clean-up process.
- if args.processes > 1:
- signal.signal(signal.SIGTERM, lambda *args: sys.exit(-signal.SIGTERM))
- return results
+def remove_prefix(s, sub):
+ """Remove everything before last occurrence of substring sub in string s."""
+ if sub in s:
+ inc_sub = s.rfind(sub)
+ return s[inc_sub:]
+ return s
-def parallel_classify_warnings(warning_lines, parallel_process):
- """Classify all warning lines with num_cpu parallel processes."""
- num_cpu = args.processes
- if num_cpu > 1:
- groups = [[] for x in range(num_cpu)]
- i = 0
- for x in warning_lines:
- groups[i].append(x)
- i = (i + 1) % num_cpu
- group_results = parallel_process(num_cpu, classify_warnings, groups)
- else:
- group_results = [classify_warnings(warning_lines)]
+# TODO(emmavukelj): Don't have any generate_*_cs_link functions call
+# normalize_path a second time (the first time being in parse_input_file)
+def generate_cs_link(warning_line, flags, android_root=None):
+ if flags.platform == 'chrome':
+ return generate_chrome_cs_link(warning_line, flags)
+ if flags.platform == 'android':
+ return generate_android_cs_link(warning_line, flags, android_root)
+ return 'https://cs.corp.google.com/'
- for result in group_results:
- for line, pattern_idx, project_idx in result:
- pattern = warn_patterns[pattern_idx]
- pattern['members'].append(line)
- message_idx = len(warning_messages)
- warning_messages.append(line)
- warning_records.append([pattern_idx, project_idx, message_idx])
- pname = '???' if project_idx < 0 else project_names[project_idx]
- # Count warnings by project.
- if pname in pattern['projects']:
- pattern['projects'][pname] += 1
- else:
- pattern['projects'][pname] = 1
+
+def generate_android_cs_link(warning_line, flags, android_root):
+ """Generate the code search link for a warning line in Android."""
+ # max_splits=2 -> only 3 items
+ raw_path, line_number_str, _ = warning_line.split(':', 2)
+ normalized_path = normalize_path(raw_path, flags, android_root)
+ if not flags.url:
+ return normalized_path
+ link_path = flags.url + '/' + normalized_path
+ if line_number_str.isdigit():
+ link_path += flags.separator + line_number_str
+ return link_path
+
+
+def generate_chrome_cs_link(warning_line, flags):
+ """Generate the code search link for a warning line in Chrome."""
+ split_line = warning_line.split(':')
+ raw_path = split_line[0]
+ normalized_path = normalize_path(raw_path, flags)
+ link_base = 'https://cs.chromium.org/'
+ link_add = 'chromium'
+ link_path = None
+
+ # Basically just going through a few specific directory cases and specifying
+ # the proper behavior for that case. This list of cases was accumulated
+ # through trial and error manually going through the warnings.
+ #
+ # This code pattern of using case-specific "if"s instead of "elif"s looks
+ # possibly accidental and mistaken but it is intentional because some paths
+ # fall under several cases (e.g. third_party/lib/nghttp2_frame.c) and for
+ # those we want the most specific case to be applied. If there is reliable
+ # knowledge of exactly where these occur, this could be changed to "elif"s
+ # but there is no reliable set of paths falling under multiple cases at the
+ # moment.
+ if '/src/third_party' in raw_path:
+ link_path = remove_prefix(raw_path, '/src/third_party/')
+ if '/chrome_root/src_internal/' in raw_path:
+ link_path = remove_prefix(raw_path, '/chrome_root/src_internal/')
+ link_path = link_path[len('/chrome_root'):] # remove chrome_root
+ if '/chrome_root/src/' in raw_path:
+ link_path = remove_prefix(raw_path, '/chrome_root/src/')
+ link_path = link_path[len('/chrome_root'):] # remove chrome_root
+ if '/libassistant/' in raw_path:
+ link_add = 'eureka_internal/chromium/src'
+ link_base = 'https://cs.corp.google.com/' # internal data
+ link_path = remove_prefix(normalized_path, '/libassistant/')
+ if raw_path.startswith('gen/'):
+ link_path = '/src/out/Debug/gen/' + normalized_path
+ if '/gen/' in raw_path:
+ return '%s?q=file:%s' % (link_base, remove_prefix(normalized_path, '/gen/'))
+
+ if not link_path and (raw_path.startswith('src/') or
+ raw_path.startswith('src_internal/')):
+ link_path = '/%s' % raw_path
+
+ if not link_path: # can't find specific link, send a query
+ return '%s?q=file:%s' % (link_base, normalized_path)
+
+ line_number = int(split_line[1])
+ link = '%s%s%s?l=%d' % (link_base, link_add, link_path, line_number)
+ return link
def find_warn_py_and_android_root(path):
- """Set and return android_root path if it is found."""
- global android_root
+ """Return android source root path if warn.py is found."""
parts = path.split('/')
for idx in reversed(range(2, len(parts))):
root_path = '/'.join(parts[:idx])
# Android root directory should contain this script.
if os.path.exists(root_path + '/build/make/tools/warn.py'):
- android_root = root_path
- return True
- return False
+ return root_path
+ return ''
-def find_android_root():
- """Guess android_root from common prefix of file paths."""
+def find_android_root(buildlog):
+ """Guess android source root from common prefix of file paths."""
# Use the longest common prefix of the absolute file paths
# of the first 10000 warning messages as the android_root.
- global android_root
- warning_lines = set()
+ warning_lines = []
warning_pattern = re.compile('^/[^ ]*/[^ ]*: warning: .*')
count = 0
- infile = io.open(args.buildlog, mode='r', encoding='utf-8')
- for line in infile:
+ for line in buildlog:
if warning_pattern.match(line):
- warning_lines.add(line)
+ warning_lines.append(line)
count += 1
if count > 9999:
break
@@ -468,56 +579,110 @@
# the source tree root.
if count < 100:
path = os.path.normpath(re.sub(':.*$', '', line))
- if find_warn_py_and_android_root(path):
- return
+ android_root = find_warn_py_and_android_root(path)
+ if android_root:
+ return android_root
# Do not use common prefix of a small number of paths.
if count > 10:
# pytype: disable=wrong-arg-types
root_path = os.path.commonprefix(warning_lines)
# pytype: enable=wrong-arg-types
if len(root_path) > 2 and root_path[len(root_path) - 1] == '/':
- android_root = root_path[:-1]
+ return root_path[:-1]
+ return ''
-def remove_android_root_prefix(path):
+def remove_android_root_prefix(path, android_root):
"""Remove android_root prefix from path if it is found."""
if path.startswith(android_root):
return path[1 + len(android_root):]
+ return path
+
+
+def normalize_path(path, flags, android_root=None):
+ """Normalize file path relative to src/ or src-internal/ directory."""
+ path = os.path.normpath(path)
+
+ if flags.platform == 'android':
+ if android_root:
+ return remove_android_root_prefix(path, android_root)
+ return path
+
+ # Remove known prefix of root path and normalize the suffix.
+ idx = path.find('chrome_root/')
+ if idx >= 0:
+ # remove chrome_root/, we want path relative to that
+ return path[idx + len('chrome_root/'):]
else:
return path
-def normalize_path(path):
- """Normalize file path relative to android_root."""
- # If path is not an absolute path, just normalize it.
- path = os.path.normpath(path)
- # Remove known prefix of root path and normalize the suffix.
- if path[0] == '/' and android_root:
- return remove_android_root_prefix(path)
- return path
-
-
-def normalize_warning_line(line):
- """Normalize file path relative to android_root in a warning line."""
- # replace fancy quotes with plain ol' quotes
+def normalize_warning_line(line, flags, android_root=None):
+ """Normalize file path relative to src directory in a warning line."""
line = re.sub(u'[\u2018\u2019]', '\'', line)
# replace non-ASCII chars to spaces
line = re.sub(u'[^\x00-\x7f]', ' ', line)
line = line.strip()
first_column = line.find(':')
- if first_column > 0:
- return normalize_path(line[:first_column]) + line[first_column:]
- else:
- return line
+ return normalize_path(line[:first_column], flags,
+ android_root) + line[first_column:]
-def parse_input_file(infile):
- """Parse input file, collect parameters and warning lines."""
- global android_root
- global platform_version
- global target_product
- global target_variant
- line_counter = 0
+def parse_input_file_chrome(infile, flags):
+ """Parse Chrome input file, collect parameters and warning lines."""
+ platform_version = 'unknown'
+ board_name = 'unknown'
+ architecture = 'unknown'
+
+ # only handle warning lines of format 'file_path:line_no:col_no: warning: ...'
+ chrome_warning_pattern = r'^[^ ]*/[^ ]*:[0-9]+:[0-9]+: warning: .*'
+
+ warning_pattern = re.compile(chrome_warning_pattern)
+
+ # Collect all unique warning lines
+ # Remove the duplicated warnings save ~8% of time when parsing
+ # one typical build log than before
+ unique_warnings = dict()
+ for line in infile:
+ if warning_pattern.match(line):
+ normalized_line = normalize_warning_line(line, flags)
+ if normalized_line not in unique_warnings:
+ unique_warnings[normalized_line] = generate_cs_link(line, flags)
+ elif (platform_version == 'unknown' or board_name == 'unknown' or
+ architecture == 'unknown'):
+ m = re.match(r'.+Package:.+chromeos-base/chromeos-chrome-', line)
+ if m is not None:
+ platform_version = 'R' + line.split('chrome-')[1].split('_')[0]
+ continue
+ m = re.match(r'.+Source\sunpacked\sin\s(.+)', line)
+ if m is not None:
+ board_name = m.group(1).split('/')[2]
+ continue
+ m = re.match(r'.+USE:\s*([^\s]*).*', line)
+ if m is not None:
+ architecture = m.group(1)
+ continue
+
+ header_str = '%s - %s - %s' % (platform_version, board_name, architecture)
+ return unique_warnings, header_str
+
+
+def add_normalized_line_to_warnings(line, flags, android_root, unique_warnings):
+ """Parse/normalize path, updating warning line and add to warnings dict."""
+ normalized_line = normalize_warning_line(line, flags, android_root)
+ if normalized_line not in unique_warnings:
+ unique_warnings[normalized_line] = generate_cs_link(line, flags,
+ android_root)
+ return unique_warnings
+
+
+def parse_input_file_android(infile, flags):
+ """Parse Android input file, collect parameters and warning lines."""
+ platform_version = 'unknown'
+ target_product = 'unknown'
+ target_variant = 'unknown'
+ android_root = find_android_root(infile)
+ infile.seek(0)
# rustc warning messages have two lines that should be combined:
# warning: description
@@ -532,20 +697,25 @@
warning_without_file = re.compile('^warning: .*')
rustc_file_position = re.compile('^[ ]+--> [^ ]*/[^ ]*:[0-9]+:[0-9]+')
- # Collect all warnings into the warning_lines set.
- warning_lines = set()
+ # Collect all unique warning lines
+ # Remove the duplicated warnings save ~8% of time when parsing
+ # one typical build log than before
+ unique_warnings = dict()
+ line_counter = 0
prev_warning = ''
for line in infile:
if prev_warning:
if rustc_file_position.match(line):
# must be a rustc warning, combine 2 lines into one warning
line = line.strip().replace('--> ', '') + ': ' + prev_warning
- warning_lines.add(normalize_warning_line(line))
+ unique_warnings = add_normalized_line_to_warnings(
+ line, flags, android_root, unique_warnings)
prev_warning = ''
continue
# add prev_warning, and then process the current line
prev_warning = 'unknown_source_file: ' + prev_warning
- warning_lines.add(normalize_warning_line(prev_warning))
+ unique_warnings = add_normalized_line_to_warnings(
+ prev_warning, flags, android_root, unique_warnings)
prev_warning = ''
if warning_pattern.match(line):
@@ -553,7 +723,8 @@
# save this line and combine it with the next line
prev_warning = line
else:
- warning_lines.add(normalize_warning_line(line))
+ unique_warnings = add_normalized_line_to_warnings(
+ line, flags, android_root, unique_warnings)
continue
if line_counter < 100:
@@ -568,17 +739,36 @@
m = re.search('(?<=^TARGET_BUILD_VARIANT=).*', line)
if m is not None:
target_variant = m.group(0)
- m = re.search('.* TOP=([^ ]*) .*', line)
+ m = re.search('(?<=^TOP=).*', line)
if m is not None:
android_root = m.group(1)
- return warning_lines
+
+ if android_root:
+ new_unique_warnings = dict()
+ for warning_line in unique_warnings:
+ normalized_line = normalize_warning_line(warning_line, flags,
+ android_root)
+ new_unique_warnings[normalized_line] = generate_android_cs_link(
+ warning_line, flags, android_root)
+ unique_warnings = new_unique_warnings
+
+ header_str = '%s - %s - %s' % (platform_version, target_product,
+ target_variant)
+ return unique_warnings, header_str
+
+
+def parse_input_file(infile, flags):
+ if flags.platform == 'chrome':
+ return parse_input_file_chrome(infile, flags)
+ if flags.platform == 'android':
+ return parse_input_file_android(infile, flags)
+ raise RuntimeError('parse_input_file not defined for platform %s' %
+ flags.platform)
# Return s with escaped backslash and quotation characters.
def escape_string(s):
- # pytype: disable=attribute-error
return s.replace('\\', '\\\\').replace('"', '\\"')
- # pytype: enable=attribute-error
# Return s without trailing '\n' and escape the quotation characters.
@@ -589,19 +779,22 @@
return escape_string(s)
-def emit_warning_array(name, writer):
+def emit_warning_array(name, writer, warn_patterns):
writer('var warning_{} = ['.format(name))
- for i in range(len(warn_patterns)):
- writer('{},'.format(warn_patterns[i][name]))
+ for w in warn_patterns:
+ if name == 'severity':
+ writer('{},'.format(w[name].value))
+ else:
+ writer('{},'.format(w[name]))
writer('];')
-def emit_warning_arrays(writer):
- emit_warning_array('severity', writer)
+def emit_warning_arrays(writer, warn_patterns):
+ emit_warning_array('severity', writer, warn_patterns)
writer('var warning_description = [')
- for i in range(len(warn_patterns)):
- if warn_patterns[i]['members']:
- writer('"{}",'.format(escape_string(warn_patterns[i]['description'])))
+ for w in warn_patterns:
+ if w['members']:
+ writer('"{}",'.format(escape_string(w['description'])))
else:
writer('"",') # no such warning
writer('];')
@@ -621,7 +814,7 @@
return byMessageCount(x1, x2);
}
const ParseLinePattern = /^([^ :]+):(\\d+):(.+)/;
- function addURL(line) {
+ function addURL(line) { // used by Android
if (FlagURL == "") return line;
if (FlagSeparator == "") {
return line.replace(ParseLinePattern,
@@ -631,6 +824,13 @@
"<a target='_blank' href='" + FlagURL + "/$1" + FlagSeparator +
"$2'>$1:$2</a>:$3");
}
+ function addURLToLine(line, link) { // used by Chrome
+ let line_split = line.split(":");
+ let path = line_split.slice(0,3).join(":");
+ let msg = line_split.slice(3).join(":");
+ let html_link = `<a target="_blank" href="${link}">${path}</a>${msg}`;
+ return html_link;
+ }
function createArrayOfDictionaries(n) {
var result = [];
for (var i=0; i<n; i++) result.push({});
@@ -690,10 +890,18 @@
"' style='display:none;'><table class='t1'>";
var c = 0;
messages.sort(compareMessages);
- for (var i=0; i<messages.length; i++) {
- result += "<tr><td class='c" + c + "'>" +
- addURL(WarningMessages[messages[i][2]]) + "</td></tr>";
- c = 1 - c;
+ if (FlagPlatform == "chrome") {
+ for (var i=0; i<messages.length; i++) {
+ result += "<tr><td class='c" + c + "'>" +
+ addURLToLine(WarningMessages[messages[i][2]], WarningLinks[messages[i][3]]) + "</td></tr>";
+ c = 1 - c;
+ }
+ } else {
+ for (var i=0; i<messages.length; i++) {
+ result += "<tr><td class='c" + c + "'>" +
+ addURL(WarningMessages[messages[i][2]]) + "</td></tr>";
+ c = 1 - c;
+ }
}
result += "</table></div>";
}
@@ -777,14 +985,14 @@
writer('];')
-def emit_js_data(writer):
+def emit_js_data(writer, flags, warning_messages, warning_links,
+ warning_records, warn_patterns, project_names):
"""Dump dynamic HTML page's static JavaScript data."""
- emit_const_string('FlagURL',
- args.url if args.url else '', writer)
- emit_const_string('FlagSeparator',
- args.separator if args.separator else '', writer)
- emit_const_string_array('SeverityColors',
- [s.color for s in Severity.levels], writer)
+ emit_const_string('FlagPlatform', flags.platform, writer)
+ emit_const_string('FlagURL', flags.url, writer)
+ emit_const_string('FlagSeparator', flags.separator, writer)
+ emit_const_string_array('SeverityColors', [s.color for s in Severity.levels],
+ writer)
emit_const_string_array('SeverityHeaders',
[s.header for s in Severity.levels], writer)
emit_const_string_array('SeverityColumnHeaders',
@@ -799,6 +1007,9 @@
writer)
emit_const_html_string_array('WarningMessages', warning_messages, writer)
emit_const_object_array('Warnings', warning_records, writer)
+ if flags.platform == 'chrome':
+ emit_const_html_string_array('WarningLinks', warning_links, writer)
+
draw_table_javascript = """
google.charts.load('current', {'packages':['table']});
@@ -822,82 +1033,201 @@
"""
-def dump_html(output_stream):
- """Dump the html output to output_stream."""
+def dump_html(flags, output_stream, warning_messages, warning_links,
+ warning_records, header_str, warn_patterns, project_names):
+ """Dump the flags output to output_stream."""
writer = make_writer(output_stream)
- dump_html_prologue('Warnings for ' + platform_version + ' - ' +
- target_product + ' - ' + target_variant, writer)
- dump_stats(writer)
+ dump_html_prologue('Warnings for ' + header_str, writer, warn_patterns,
+ project_names)
+ dump_stats(writer, warn_patterns)
writer('<br><div id="stats_table"></div><br>')
writer('\n<script>')
- emit_js_data(writer)
+ emit_js_data(writer, flags, warning_messages, warning_links, warning_records,
+ warn_patterns, project_names)
writer(scripts_for_warning_groups)
writer('</script>')
emit_buttons(writer)
# Warning messages are grouped by severities or project names.
writer('<br><div id="warning_groups"></div>')
- if args.byproject:
+ if flags.byproject:
writer('<script>groupByProject();</script>')
else:
writer('<script>groupBySeverity();</script>')
- dump_fixed(writer)
+ dump_fixed(writer, warn_patterns)
dump_html_epilogue(writer)
-##### Functions to count warnings and dump csv file. #########################
+def parse_compiler_output(compiler_output):
+ """Parse compiler output for relevant info."""
+ split_output = compiler_output.split(':', 3) # 3 = max splits
+ file_path = split_output[0]
+ line_number = int(split_output[1])
+ col_number = int(split_output[2].split(' ')[0])
+ warning_message = split_output[3]
+ return file_path, line_number, col_number, warning_message
-def description_for_csv(category):
- if not category['description']:
- return '?'
- return category['description']
-
-
-def count_severity(writer, sev, kind):
- """Count warnings of given severity."""
- total = 0
- for i in warn_patterns:
- if i['severity'] == sev and i['members']:
- n = len(i['members'])
- total += n
- warning = kind + ': ' + description_for_csv(i)
- writer.writerow([n, '', warning])
- # print number of warnings for each project, ordered by project name.
- # pytype: disable=attribute-error
- projects = sorted(i['projects'].keys())
- # pytype: enable=attribute-error
- for p in projects:
- writer.writerow([i['projects'][p], p, warning])
- writer.writerow([total, '', kind + ' warnings'])
-
- return total
-
-
-# dump number of warnings in csv format to stdout
-def dump_csv(writer):
- """Dump number of warnings in csv format to stdout."""
- sort_warnings()
- total = 0
- for s in Severity.levels:
- if s != Severity.SEVERITY_UNKNOWN:
- total += count_severity(writer, s, s.column_header)
- writer.writerow([total, '', 'All warnings'])
-
-
-def common_main(parallel_process):
- """Real main function to classify warnings and generate .html file."""
- find_android_root()
- # We must use 'utf-8' codec to parse some non-ASCII code in warnings.
- warning_lines = parse_input_file(
- io.open(args.buildlog, mode='r', encoding='utf-8'))
- parallel_classify_warnings(warning_lines, parallel_process)
- # If a user pases a csv path, save the fileoutput to the path
- # If the user also passed gencsv write the output to stdout
- # If the user did not pass gencsv flag dump the html report to stdout.
- if args.csvpath:
- with open(args.csvpath, 'w') as f:
- dump_csv(csv.writer(f, lineterminator='\n'))
- if args.gencsv:
- dump_csv(csv.writer(sys.stdout, lineterminator='\n'))
+def get_warn_patterns(platform):
+ """Get and initialize warn_patterns."""
+ warn_patterns = []
+ if platform == 'chrome':
+ warn_patterns = cpp_patterns.warn_patterns
+ elif platform == 'android':
+ warn_patterns = make_patterns.warn_patterns + cpp_patterns.warn_patterns + java_patterns.warn_patterns + tidy_patterns.warn_patterns + other_patterns.warn_patterns
else:
- dump_html(sys.stdout)
+ raise Exception('platform name %s is not valid' % platform)
+ for w in warn_patterns:
+ w['members'] = []
+ # Each warning pattern has a 'projects' dictionary, that
+ # maps a project name to number of warnings in that project.
+ w['projects'] = {}
+ return warn_patterns
+
+
+def get_project_list(platform):
+ """Return project list for appropriate platform."""
+ if platform == 'chrome':
+ return chrome_project_list.project_list
+ if platform == 'android':
+ return android_project_list.project_list
+ raise Exception('platform name %s is not valid' % platform)
+
+
+def parallel_classify_warnings(warning_data, args, project_names,
+ project_patterns, warn_patterns,
+ use_google3, create_launch_subprocs_fn,
+ classify_warnings_fn):
+ """Classify all warning lines with num_cpu parallel processes."""
+ num_cpu = args.processes
+ group_results = []
+
+ if num_cpu > 1:
+ # set up parallel processing for this...
+ warning_groups = [[] for _ in range(num_cpu)]
+ i = 0
+ for warning, link in warning_data.items():
+ warning_groups[i].append((warning, link))
+ i = (i + 1) % num_cpu
+ arg_groups = [[] for _ in range(num_cpu)]
+ for i, group in enumerate(warning_groups):
+ arg_groups[i] = [{
+ 'group': group,
+ 'project_patterns': project_patterns,
+ 'warn_patterns': warn_patterns,
+ 'num_processes': num_cpu
+ }]
+
+ group_results = create_launch_subprocs_fn(num_cpu,
+ classify_warnings_fn,
+ arg_groups,
+ group_results)
+ else:
+ group_results = []
+ for warning, link in warning_data.items():
+ classify_one_warning(warning, link, group_results,
+ project_patterns, warn_patterns)
+ group_results = [group_results]
+
+ warning_messages = []
+ warning_links = []
+ warning_records = []
+ if use_google3:
+ group_results = [group_results]
+ for group_result in group_results:
+ for result in group_result:
+ for line, link, pattern_idx, project_idx in result:
+ pattern = warn_patterns[pattern_idx]
+ pattern['members'].append(line)
+ message_idx = len(warning_messages)
+ warning_messages.append(line)
+ link_idx = len(warning_links)
+ warning_links.append(link)
+ warning_records.append([pattern_idx, project_idx, message_idx,
+ link_idx])
+ pname = '???' if project_idx < 0 else project_names[project_idx]
+ # Count warnings by project.
+ if pname in pattern['projects']:
+ pattern['projects'][pname] += 1
+ else:
+ pattern['projects'][pname] = 1
+ return warning_messages, warning_links, warning_records
+
+
+def write_html(flags, project_names, warn_patterns, html_path, warning_messages,
+ warning_links, warning_records, header_str):
+ """Write warnings html file."""
+ if html_path:
+ with open(html_path, 'w') as f:
+ dump_html(flags, f, warning_messages, warning_links, warning_records,
+ header_str, warn_patterns, project_names)
+
+
+def write_out_csv(flags, warn_patterns, warning_messages, warning_links,
+ warning_records, header_str, project_names):
+ """Write warnings csv file."""
+ if flags.csvpath:
+ with open(flags.csvpath, 'w') as f:
+ dump_csv(csv.writer(f, lineterminator='\n'), warn_patterns)
+
+ if flags.gencsv:
+ dump_csv(csv.writer(sys.stdout, lineterminator='\n'), warn_patterns)
+ else:
+ dump_html(flags, sys.stdout, warning_messages, warning_links,
+ warning_records, header_str, warn_patterns, project_names)
+
+
+def process_log(logfile, flags, project_names, project_patterns, warn_patterns,
+ html_path, use_google3, create_launch_subprocs_fn,
+ classify_warnings_fn, logfile_object):
+ # pylint: disable=g-doc-args
+ # pylint: disable=g-doc-return-or-yield
+ """Function that handles processing of a log.
+
+ This is isolated into its own function (rather than just taking place in main)
+ so that it can be used by both warn.py and the borg job process_gs_logs.py, to
+ avoid duplication of code.
+ Note that if the arguments to this function change, process_gs_logs.py must
+ be updated accordingly.
+ """
+ if logfile_object is None:
+ with io.open(logfile, encoding='utf-8') as log:
+ warning_lines_and_links, header_str = parse_input_file(log, flags)
+ else:
+ warning_lines_and_links, header_str = parse_input_file(
+ logfile_object, flags)
+ warning_messages, warning_links, warning_records = parallel_classify_warnings(
+ warning_lines_and_links, flags, project_names, project_patterns,
+ warn_patterns, use_google3, create_launch_subprocs_fn,
+ classify_warnings_fn)
+
+ write_html(flags, project_names, warn_patterns, html_path,
+ warning_messages, warning_links, warning_records,
+ header_str)
+
+ return warning_messages, warning_links, warning_records, header_str
+
+
+def common_main(use_google3, create_launch_subprocs_fn, classify_warnings_fn,
+ logfile_object=None):
+ """Shared main function for Google3 and non-Google3 versions of warn.py."""
+ flags = parse_args(use_google3)
+ warn_patterns = get_warn_patterns(flags.platform)
+ project_list = get_project_list(flags.platform)
+
+ project_names = get_project_names(project_list)
+ project_patterns = [re.compile(p[1]) for p in project_list]
+
+ # html_path=None because we output html below if not outputting CSV
+ warning_messages, warning_links, warning_records, header_str = process_log(
+ logfile=flags.log, flags=flags, project_names=project_names,
+ project_patterns=project_patterns, warn_patterns=warn_patterns,
+ html_path=None, use_google3=use_google3,
+ create_launch_subprocs_fn=create_launch_subprocs_fn,
+ classify_warnings_fn=classify_warnings_fn,
+ logfile_object=logfile_object)
+
+ write_out_csv(flags, warn_patterns, warning_messages, warning_links,
+ warning_records, header_str, project_names)
+
+ # Return these values, so that caller can use them, if desired.
+ return flags, warning_messages, warning_records, warn_patterns
