[autotest] Only try to symbolicate crash dump when a crashserver is available.

Limit the number of symbolication in crashservers by allocating a crashserver
with enough capacity. If none is found, fail symbolicate_minidump_with_devserver
call with dev_server.DevServerException.

BUG=chromium:489845
TEST=local run
import common
from autotest_lib.client.common_lib.cros import dev_server
print dev_server.get_least_loaded_devserver(devserver_type=dev_server.CrashServer)
http://100.107.160.5:8082

Change-Id: I71e890c8a058c174a11c928f260bb0b0c45df938
Reviewed-on: https://chromium-review.googlesource.com/295168
Tested-by: Dan Shi <[email protected]>
Reviewed-by: Fang Deng <[email protected]>
Commit-Queue: Dan Shi <[email protected]>
Trybot-Ready: Dan Shi <[email protected]>
diff --git a/server/site_crashcollect.py b/server/site_crashcollect.py
index 02ab900..f41d769 100644
--- a/server/site_crashcollect.py
+++ b/server/site_crashcollect.py
@@ -6,13 +6,16 @@
 import os
 import re
 from autotest_lib.client.common_lib import utils as client_utils
-from autotest_lib.client.common_lib import error
 from autotest_lib.client.common_lib.cros import dev_server
+from autotest_lib.client.common_lib.cros.graphite import autotest_stats
 from autotest_lib.client.cros import constants
 from autotest_lib.server.cros.dynamic_suite.constants import JOB_BUILD_KEY
 from autotest_lib.server import utils
 
 
+CRASH_SERVER_OVERLOAD = 'crash_server_overload'
+CRASH_SERVER_FOUND = 'crash_server_found'
+
 def generate_minidump_stacktrace(minidump_path):
     """
     Generates a stacktrace for the specified minidump.
@@ -46,7 +49,15 @@
         raise dev_server.DevServerException(
             'Cannot determine build being tested.')
 
-    devserver = dev_server.CrashServer.resolve(keyvals[JOB_BUILD_KEY])
+    crashserver_name = dev_server.get_least_loaded_devserver(
+            devserver_type=dev_server.CrashServer)
+    if not crashserver_name:
+        autotest_stats.Counter(CRASH_SERVER_OVERLOAD).increment()
+        raise dev_server.DevServerException(
+                'No crash server has the capacity to symbolicate the dump.')
+    else:
+        autotest_stats.Counter(CRASH_SERVER_FOUND).increment()
+    devserver = dev_server.CrashServer(crashserver_name)
     trace_text = devserver.symbolicate_dump(
         minidump_path, keyvals[JOB_BUILD_KEY])
     if not trace_text: