auto_delete_nightly_test_data.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""A crontab script to delete night test data."""

from __future__ import print_function

__author__ = 'shenhan@google.com (Han Shen)'

import argparse
import datetime
import os
import re
import shutil
import shlex
import sys
import time

from cros_utils import command_executer
from cros_utils import constants
from cros_utils import misc

DIR_BY_WEEKDAY = ('Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun')
NIGHTLY_TESTS_WORKSPACE = os.path.join(constants.CROSTC_WORKSPACE,
                                       'nightly-tests')


def CleanNumberedDir(s, dry_run=False):
  """Deleted directories under each dated_dir."""
  chromeos_dirs = [
      os.path.join(s, x)
      for x in os.listdir(s)
      if misc.IsChromeOsTree(os.path.join(s, x))
  ]
  ce = command_executer.GetCommandExecuter(log_level='none')
  all_succeeded = True
  for cd in chromeos_dirs:
    if misc.DeleteChromeOsTree(cd, dry_run=dry_run):
      print('Successfully removed chromeos tree "{0}".'.format(cd))
    else:
      all_succeeded = False
      print('Failed to remove chromeos tree "{0}", please check.'.format(cd))

  if not all_succeeded:
    print('Failed to delete at least one chromeos tree, please check.')
    return False

  ## Now delete the numbered dir Before forcibly removing the directory, just
  ## check 's' to make sure it matches the expected pattern.  A valid dir to be
  ## removed must be '/usr/local/google/crostc/(SUN|MON|TUE...|SAT)'.
  valid_dir_pattern = ('^' + NIGHTLY_TESTS_WORKSPACE + '/(' +
                       '|'.join(DIR_BY_WEEKDAY) + ')')
  if not re.search(valid_dir_pattern, s):
    print('Trying to delete an invalid dir "{0}" (must match "{1}"), '
          'please check.'.format(s, valid_dir_pattern))
    return False

  cmd = 'rm -fr {0}'.format(s)
  if dry_run:
    print(cmd)
  else:
    if ce.RunCommand(cmd, print_to_console=False, terminated_timeout=480) == 0:
      print('Successfully removed "{0}".'.format(s))
    else:
      all_succeeded = False
      print('Failed to remove "{0}", please check.'.format(s))
  return all_succeeded


def CleanDatedDir(dated_dir, dry_run=False):
  # List subdirs under dir
  subdirs = [
      os.path.join(dated_dir, x)
      for x in os.listdir(dated_dir)
      if os.path.isdir(os.path.join(dated_dir, x))
  ]
  all_succeeded = True
  for s in subdirs:
    if not CleanNumberedDir(s, dry_run):
      all_succeeded = False
  return all_succeeded


def ProcessArguments(argv):
  """Process arguments."""
  parser = argparse.ArgumentParser(
      description='Automatically delete nightly test data directories.',
      usage='auto_delete_nightly_test_data.py options')
  parser.add_argument(
      '-d',
      '--dry_run',
      dest='dry_run',
      default=False,
      action='store_true',
      help='Only print command line, do not execute anything.')
  parser.add_argument(
      '--days_to_preserve',
      dest='days_to_preserve',
      default=3,
      help=('Specify the number of days (not including today),'
            ' test data generated on these days will *NOT* be '
            'deleted. Defaults to 3.'))
  options = parser.parse_args(argv)
  return options


def CleanChromeOsTmpFiles(chroot_tmp, days_to_preserve, dry_run):
  rv = 0
  ce = command_executer.GetCommandExecuter()
  # Clean chroot/tmp/test_that_* and chroot/tmp/tmpxxxxxx, that were last
  # accessed more than specified time.
  minutes = 1440 * days_to_preserve
  cmd = (r'find {0} -maxdepth 1 -type d '
         r'\( -name "test_that_*"  -amin +{1} -o '
         r'   -name "cros-update*" -amin +{1} -o '
         r'   -name "CrAU_temp_data*" -amin +{1} -o '
         r' -regex "{0}/tmp......" -amin +{1} \) '
         r'-exec bash -c "echo rm -fr {{}}" \; '
         r'-exec bash -c "rm -fr {{}}" \;').format(chroot_tmp, minutes)
  if dry_run:
    print('Going to execute:\n%s' % cmd)
  else:
    rv = ce.RunCommand(cmd, print_to_console=False)
    if rv == 0:
      print('Successfully cleaned chromeos tree tmp directory '
            '"{0}".'.format(chroot_tmp))
    else:
      print('Some directories were not removed under chromeos tree '
            'tmp directory -"{0}".'.format(chroot_tmp))

  return rv


def CleanChromeOsImageFiles(chroot_tmp, subdir_suffix, days_to_preserve,
                            dry_run):
  # Clean files that were last accessed more than the specified time.
  seconds_delta = days_to_preserve * 24 * 3600
  now = time.time()
  errors = 0

  for tmp_dir in os.listdir(chroot_tmp):
    # Directory under /tmp
    tmp_dir = os.path.join(chroot_tmp, tmp_dir)
    if tmp_dir.endswith(subdir_suffix):
      # Tmp directory which ends with subdir_suffix.
      for subdir in os.listdir(tmp_dir):
        # Subdirectories targeted for deletion.
        subdir_path = os.path.join(tmp_dir, subdir)
        if now - os.path.getatime(subdir_path) > seconds_delta:
          if dry_run:
            print('Will run:\nshutil.rmtree({})'.format(subdir_path))
          else:
            try:
              shutil.rmtree(subdir_path)
              print('Successfully cleaned chromeos image autotest directories '
                    'from "{}".'.format(subdir_path))
            except OSError:
              print('Some image autotest directories were not removed from '
                    '"{}".'.format(subdir_path))
              errors += 1

  return errors


def CleanChromeOsTmpAndImages(days_to_preserve=1, dry_run=False):
  """Delete temporaries, images under crostc/chromeos."""
  chromeos_chroot_tmp = os.path.join(constants.CROSTC_WORKSPACE, 'chromeos',
                                     'chroot', 'tmp')
  # Clean files in tmp directory
  rv = CleanChromeOsTmpFiles(chromeos_chroot_tmp, days_to_preserve, dry_run)
  # Clean image files in *-tryjob directories
  rv += CleanChromeOsImageFiles(chromeos_chroot_tmp, '-tryjob',
                                days_to_preserve, dry_run)
  # Clean image files in *-release directories
  rv += CleanChromeOsImageFiles(chromeos_chroot_tmp, '-release',
                                days_to_preserve, dry_run)
  # Clean image files in *-pfq directories
  rv += CleanChromeOsImageFiles(chromeos_chroot_tmp, '-pfq', days_to_preserve,
                                dry_run)
  # Clean image files in *-llvm-next-nightly directories
  rv += CleanChromeOsImageFiles(chromeos_chroot_tmp, '-llvm-next-nightly',
                                days_to_preserve, dry_run)

  return rv


def CleanOldCLs(days_to_preserve='1', dry_run=False):
  """Abandon old CLs created by automation tooling."""
  ce = command_executer.GetCommandExecuter()
  chromeos_root = os.path.join(constants.CROSTC_WORKSPACE, 'chromeos')
  # Find Old CLs.
  old_cls_cmd = ('gerrit --raw search "owner:me status:open age:%sd"' %
                 days_to_preserve)
  _, cls, _ = ce.ChrootRunCommandWOutput(
      chromeos_root, old_cls_cmd, print_to_console=False)
  # Convert any whitespaces to spaces.
  cls = ' '.join(cls.split())
  if not cls:
    return 0

  abandon_cls_cmd = ('gerrit abandon %s' % cls)
  if dry_run:
    print('Going to execute: %s' % abandon_cls_cmd)
    return 0

  return ce.ChrootRunCommand(
      chromeos_root, abandon_cls_cmd, print_to_console=False)


def CleanChromeTelemetryTmpFiles(dry_run):
  rv = 0
  ce = command_executer.GetCommandExecuter()
  tmp_dir = os.path.join(constants.CROSTC_WORKSPACE, 'chromeos', '.cache',
                         'distfiles', 'chrome-src-internal', 'src', 'tmp')
  cmd = f'rm -fr {shlex.quote(tmp_dir)}/tmp*telemetry_Crosperf'
  if dry_run:
    print(f'Going to execute:\n{cmd}')
  else:
    rv = ce.RunCommand(cmd, print_to_console=False)
    if rv == 0:
      print(f'Successfully cleaned chrome tree tmp directory ' f'{tmp_dir!r} .')
    else:
      print(f'Some directories were not removed under chrome tree '
            f'tmp directory {tmp_dir!r}.')
  return rv


def Main(argv):
  """Delete nightly test data directories, tmps and test images."""
  options = ProcessArguments(argv)
  # Function 'isoweekday' returns 1(Monday) - 7 (Sunday).
  d = datetime.datetime.today().isoweekday()
  # We go back 1 week, delete from that day till we are
  # options.days_to_preserve away from today.
  s = d - 7
  e = d - int(options.days_to_preserve)
  rv = 0
  for i in range(s + 1, e):
    if i <= 0:
      ## Wrap around if index is negative.  6 is from i + 7 - 1, because
      ## DIR_BY_WEEKDAY starts from 0, while isoweekday is from 1-7.
      dated_dir = DIR_BY_WEEKDAY[i + 6]
    else:
      dated_dir = DIR_BY_WEEKDAY[i - 1]

    rv += 0 if CleanDatedDir(
        os.path.join(NIGHTLY_TESTS_WORKSPACE, dated_dir),
        options.dry_run) else 1

  ## Clean temporaries, images under crostc/chromeos
  rv2 = CleanChromeOsTmpAndImages(
      int(options.days_to_preserve), options.dry_run)

  # Clean CLs that are not updated in last 2 weeks.
  rv3 = CleanOldCLs('14', options.dry_run)

  # Clean telemetry temporaries from chrome source tree inside chroot.
  rv4 = CleanChromeTelemetryTmpFiles(options.dry_run)

  return rv + rv2 + rv3 + rv4


if __name__ == '__main__':
  retval = Main(sys.argv[1:])
  sys.exit(retval)