[autotest] Add actions to take when add/remove role from server
This CL adds actions needed to make adding/removing role from server effective.
For example, when server database is enabled and a new drone is added, scheduler
needs to be restarted.
BUG=chromium:424778
CQ-DEPEND=CL:232003
TEST=unittest, setup local server database, manually test follow cases
(CL 232525 is needed for drone test)
add scheduler: Confirm scheduler service is started in the server.
remove scheduler: Confirm scheduler service is stopped in the server.
add drone: Confirm scheduler service is restarted.
remove drone: Confirm scheduler service is restarted
Change-Id: I14d6bb15d68a9b94fa3ab5b0bcc202469b253c89
Reviewed-on: https://chromium-review.googlesource.com/233181
Tested-by: Dan Shi <[email protected]>
Trybot-Ready: Dan Shi <[email protected]>
Reviewed-by: Fang Deng <[email protected]>
Commit-Queue: Dan Shi <[email protected]>
diff --git a/site_utils/server_manager_utils.py b/site_utils/server_manager_utils.py
new file mode 100644
index 0000000..21c1088
--- /dev/null
+++ b/site_utils/server_manager_utils.py
@@ -0,0 +1,271 @@
+# Copyright 2014 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""This module provides utility functions to help managing servers in server
+database (defined in global config section AUTOTEST_SERVER_DB).
+
+"""
+
+import subprocess
+import sys
+
+import common
+
+import django.core.exceptions
+from autotest_lib.client.common_lib.global_config import global_config
+from autotest_lib.frontend.server import models as server_models
+from autotest_lib.site_utils.lib import infra
+
+
+class ServerActionError(Exception):
+ """Exception raised when action on server failed.
+ """
+
+
+def use_server_db():
+ """Check if use_server_db is enabled in configuration.
+
+ @return: True if use_server_db is set to True in global config.
+ """
+ return global_config.get_config_value(
+ 'SERVER', 'use_server_db', default=False, type=bool)
+
+
+def warn_missing_role(role, exclude_server):
+ """Post a warning if Autotest instance has no other primary server with
+ given role.
+
+ @param role: Name of the role.
+ @param exclude_server: Server to be excluded from search for role.
+ """
+ servers = server_models.Server.objects.filter(
+ roles__role=role,
+ status=server_models.Server.STATUS.PRIMARY).exclude(
+ hostname=exclude_server.hostname)
+ if not servers:
+ message = ('WARNING! There will be no server with role %s after it\'s '
+ 'removed from server %s. Autotest will not function '
+ 'normally without any server in role %s.' %
+ (role, exclude_server.hostname, role))
+ print >> sys.stderr, message
+
+
+def get_servers(hostname=None, role=None, status=None):
+ """Find servers with given role and status.
+
+ @param hostname: hostname of the server.
+ @param role: Role of server, default to None.
+ @param status: Status of server, default to None.
+
+ @return: A list of server objects with given role and status.
+ """
+ filters = {}
+ if hostname:
+ filters['hostname'] = hostname
+ if role:
+ filters['roles__role'] = role
+ if status:
+ filters['status'] = status
+ return list(server_models.Server.objects.filter(**filters))
+
+
+def get_server_details(servers, table=False, summary=False):
+ """Get a string of given servers' details.
+
+ The method can return a string of server information in 3 different formats:
+ A detail view:
+ Hostname : server2
+ Status : primary
+ Roles : drone
+ Attributes : {'max_processes':300}
+ Date Created : 2014-11-25 12:00:00
+ Date Modified: None
+ Note : Drone in lab1
+ A table view:
+ Hostname | Status | Roles | Date Created | Date Modified | Note
+ server1 | backup | scheduler | 2014-11-25 23:45:19 | |
+ server2 | primary | drone | 2014-11-25 12:00:00 | | Drone
+ A summary view:
+ scheduler : server1(backup), server3(primary),
+ host_scheduler :
+ drone : server2(primary),
+ devserver :
+ database :
+ suite_scheduler:
+ crash_server :
+ No Role :
+
+ The method returns detail view of each server and a summary view by default.
+ If `table` is set to True, only table view will be returned.
+ If `summary` is set to True, only summary view will be returned.
+
+ @param servers: A list of servers to get details.
+ @param table: True to return a table view instead of a detail view,
+ default is set to False.
+ @param summary: True to only show the summary of roles and status of
+ given servers.
+
+ @return: A string of the information of given servers.
+ """
+ # Format string to display a table view.
+ # Hostname, Status, Roles, Date Created, Date Modified, Note
+ TABLEVIEW_FORMAT = ('%(hostname)-30s | %(status)-7s | %(roles)-20s | '
+ '%(date_created)-19s | %(date_modified)-19s | %(note)s')
+
+ result = ''
+ if not table and not summary:
+ for server in servers:
+ result += '\n' + str(server)
+ elif table:
+ result += (TABLEVIEW_FORMAT %
+ {'hostname':'Hostname', 'status':'Status',
+ 'roles':'Roles', 'date_created':'Date Created',
+ 'date_modified':'Date Modified', 'note':'Note'})
+ for server in servers:
+ roles = ','.join(server.get_role_names())
+ result += '\n' + (TABLEVIEW_FORMAT %
+ {'hostname':server.hostname,
+ 'status': server.status or '',
+ 'roles': roles,
+ 'date_created': server.date_created,
+ 'date_modified': server.date_modified or '',
+ 'note': server.note or ''})
+ elif summary:
+ result += 'Roles and status of servers:\n\n'
+ for role, _ in server_models.ServerRole.ROLE.choices():
+ servers_of_role = [s for s in servers if role in
+ [r.role for r in s.roles.all()]]
+ result += '%-15s: ' % role
+ for server in servers_of_role:
+ result += '%s(%s), ' % (server.hostname, server.status)
+ result += '\n'
+ servers_without_role = [s.hostname for s in servers
+ if not s.roles.all()]
+ result += '%-15s: %s' % ('No Role', ', '.join(servers_without_role))
+
+ return result
+
+
+def check_server(hostname, role):
+ """Confirm server with given hostname is ready to be primary of given role.
+
+ If the server is a backup and failed to be verified for the role, remove
+ the role from its roles list. If it has no other role, set its status to
+ repair_required.
+
+ @param hostname: hostname of the server.
+ @param role: Role to be checked.
+ @return: True if server can be verified for the given role, otherwise
+ return False.
+ """
+ # TODO(dshi): Add more logic to confirm server is ready for the role.
+ # For now, the function just checks if server is ssh-able.
+ try:
+ infra.execute_command(hostname, 'true')
+ return True
+ except subprocess.CalledProcessError as e:
+ print >> sys.stderr, ('Failed to check server %s, error: %s' %
+ (hostname, e))
+ return False
+
+
+def verify_server(exist=True):
+ """Decorator to check if server with given hostname exists in the database.
+
+ @param exist: Set to True to confirm server exists in the database, raise
+ exception if not. If it's set to False, raise exception if
+ server exists in database. Default is True.
+
+ @raise ServerActionError: If `exist` is True and server does not exist in
+ the database, or `exist` is False and server exists
+ in the database.
+ """
+ def deco_verify(func):
+ """Wrapper for the decorator.
+
+ @param func: Function to be called.
+ """
+ def func_verify(*args, **kwargs):
+ """Decorator to check if server exists.
+
+ If exist is set to True, raise ServerActionError is server with
+ given hostname is not found in server database.
+ If exist is set to False, raise ServerActionError is server with
+ given hostname is found in server database.
+
+ @param func: function to be called.
+ @param args: arguments for function to be called.
+ @param kwargs: keyword arguments for function to be called.
+ """
+ hostname = kwargs['hostname']
+ try:
+ server = server_models.Server.objects.get(hostname=hostname)
+ except django.core.exceptions.ObjectDoesNotExist:
+ server = None
+
+ if not exist and server:
+ raise ServerActionError('Server %s already exists.' %
+ hostname)
+ if exist and not server:
+ raise ServerActionError('Server %s does not exist in the '
+ 'database.' % hostname)
+ if server:
+ kwargs['server'] = server
+ return func(*args, **kwargs)
+ return func_verify
+ return deco_verify
+
+
+def get_drones():
+ """Get a list of drones in status primary.
+
+ @return: A list of drones in status primary.
+ """
+ servers = get_servers(role=server_models.ServerRole.ROLE.DRONE,
+ status=server_models.Server.STATUS.PRIMARY)
+ return [s.hostname for s in servers]
+
+
+def delete_attribute(server, attribute):
+ """Delete the attribute from the host.
+
+ @param server: An object of server_models.Server.
+ @param attribute: Name of an attribute of the server.
+ """
+ attributes = server.attributes.filter(attribute=attribute)
+ if not attributes:
+ raise ServerActionError('Server %s does not have attribute %s' %
+ (server.hostname, attribute))
+ attributes[0].delete()
+ print 'Attribute %s is deleted from server %s.' % (attribute,
+ server.hostname)
+
+
+def change_attribute(server, attribute, value):
+ """Change the value of an attribute of the server.
+
+ @param server: An object of server_models.Server.
+ @param attribute: Name of an attribute of the server.
+ @param value: Value of the attribute of the server.
+
+ @raise ServerActionError: If the attribute already exists and has the
+ given value.
+ """
+ attributes = server_models.ServerAttribute.objects.filter(
+ server=server, attribute=attribute)
+ if attributes and attributes[0].value == value:
+ raise ServerActionError('Attribute %s for Server %s already has '
+ 'value of %s.' %
+ (attribute, server.hostname, value))
+ if attributes:
+ old_value = attributes[0].value
+ attributes[0].value = value
+ attributes[0].save()
+ print ('Attribute `%s` of server %s is changed from %s to %s.' %
+ (attribute, server.hostname, old_value, value))
+ else:
+ server_models.ServerAttribute.objects.create(
+ server=server, attribute=attribute, value=value)
+ print ('Attribute `%s` of server %s is set to %s.' %
+ (attribute, server.hostname, value))
