1#!/usr/bin/env python3 2# Copyright 2014 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Report whether DUTs are working or broken. 7 8usage: dut_status [ <options> ] [hostname ...] 9 10Reports on the history and status of selected DUT hosts, to 11determine whether they're "working" or "broken". For purposes of 12the script, "broken" means "the DUT requires manual intervention 13before it can be used for further testing", and "working" means "not 14broken". The status determination is based on the history of 15completed jobs for the DUT in a given time interval; still-running 16jobs are not considered. 17 18Time Interval Selection 19~~~~~~~~~~~~~~~~~~~~~~~ 20A DUT's reported status is based on the DUT's job history in a time 21interval determined by command line options. The interval is 22specified with up to two of three options: 23 --until/-u DATE/TIME - Specifies an end time for the search 24 range. (default: now) 25 --since/-s DATE/TIME - Specifies a start time for the search 26 range. (no default) 27 --duration/-d HOURS - Specifies the length of the search interval 28 in hours. (default: 24 hours) 29 30Any two time options completely specify the time interval. If only 31one option is provided, these defaults are used: 32 --until - Use the given end time with the default duration. 33 --since - Use the given start time with the default end time. 34 --duration - Use the given duration with the default end time. 35 36If no time options are given, use the default end time and duration. 37 38DATE/TIME values are of the form '2014-11-06 17:21:34'. 39 40DUT Selection 41~~~~~~~~~~~~~ 42By default, information is reported for DUTs named as command-line 43arguments. Options are also available for selecting groups of 44hosts: 45 --board/-b BOARD - Only include hosts with the given board. 46 --pool/-p POOL - Only include hosts in the given pool. The user 47 might be interested in the following pools: bvt, cq, 48 continuous, cts, or suites. 49 50 51The selected hosts may also be filtered based on status: 52 -w/--working - Only include hosts in a working state. 53 -n/--broken - Only include hosts in a non-working state. Hosts 54 with no job history are considered non-working. 55 56Output Formats 57~~~~~~~~~~~~~~ 58There are four available output formats: 59 * A simple list of host names. 60 * A status summary showing one line per host. 61 * A detailed job history for all selected DUTs, sorted by 62 time of execution. 63 * A job history for all selected DUTs showing only the history 64 surrounding the DUT's last change from working to broken, 65 or vice versa. 66 67The default format depends on whether hosts are filtered by 68status: 69 * With the --working or --broken options, the list of host names 70 is the default format. 71 * Without those options, the default format is the one-line status 72 summary. 73 74These options override the default formats: 75 -o/--oneline - Use the one-line summary with the --working or 76 --broken options. 77 -f/--full_history - Print detailed per-host job history. 78 -g/--diagnosis - Print the job history surrounding a status 79 change. 80 81Examples 82~~~~~~~~ 83 $ dut_status chromeos2-row4-rack2-host12 84 hostname S last checked URL 85 chromeos2-row4-rack2-host12 NO 2014-11-06 15:25:29 http://... 86 87'NO' means the DUT is broken. That diagnosis is based on a job that 88failed: 'last checked' is the time of the failed job, and the URL 89points to the job's logs. 90 91 $ dut_status.py -u '2014-11-06 15:30:00' -d 1 -f chromeos2-row4-rack2-host12 92 chromeos2-row4-rack2-host12 93 2014-11-06 15:25:29 NO http://... 94 2014-11-06 14:44:07 -- http://... 95 2014-11-06 14:42:56 OK http://... 96 97The times are the start times of the jobs; the URL points to the 98job's logs. The status indicates the working or broken status after 99the job: 100 'NO' Indicates that the DUT was believed broken after the job. 101 'OK' Indicates that the DUT was believed working after the job. 102 '--' Indicates that the job probably didn't change the DUT's 103 status. 104Typically, logs of the actual failure will be found at the last job 105to report 'OK', or the first job to report '--'. 106 107""" 108 109from __future__ import absolute_import 110from __future__ import division 111from __future__ import print_function 112 113import argparse 114import sys 115import time 116 117import common 118from autotest_lib.client.common_lib import time_utils 119from autotest_lib.server import constants 120from autotest_lib.server import frontend 121from autotest_lib.server.lib import status_history 122from autotest_lib.utils import labellib 123 124# The fully qualified name makes for lines that are too long, so 125# shorten it locally. 126HostJobHistory = status_history.HostJobHistory 127 128# _DIAGNOSIS_IDS - 129# Dictionary to map the known diagnosis codes to string values. 130 131_DIAGNOSIS_IDS = { 132 status_history.UNUSED: '??', 133 status_history.UNKNOWN: '--', 134 status_history.WORKING: 'OK', 135 status_history.BROKEN: 'NO' 136} 137 138 139# Default time interval for the --duration option when a value isn't 140# specified on the command line. 141_DEFAULT_DURATION = 24 142 143 144def _include_status(status, arguments): 145 """Determine whether the given status should be filtered. 146 147 Checks the given `status` against the command line options in 148 `arguments`. Return whether a host with that status should be 149 printed based on the options. 150 151 @param status Status of a host to be printed or skipped. 152 @param arguments Parsed arguments object as returned by 153 ArgumentParser.parse_args(). 154 155 @return Returns `True` if the command-line options call for 156 printing hosts with the status, or `False` otherwise. 157 158 """ 159 if status == status_history.WORKING: 160 return arguments.working 161 else: 162 return arguments.broken 163 164 165def _print_host_summaries(history_list, arguments): 166 """Print one-line summaries of host history. 167 168 This function handles the output format of the --oneline option. 169 170 @param history_list A list of HostHistory objects to be printed. 171 @param arguments Parsed arguments object as returned by 172 ArgumentParser.parse_args(). 173 174 """ 175 fmt = '%-30s %-2s %-19s %s' 176 print(fmt % ('hostname', 'S', 'last checked', 'URL')) 177 for history in history_list: 178 status, event = history.last_diagnosis() 179 if not _include_status(status, arguments): 180 continue 181 datestr = '---' 182 url = '---' 183 if event is not None: 184 datestr = time_utils.epoch_time_to_date_string( 185 event.start_time) 186 url = event.job_url 187 188 print(fmt % (history.hostname, 189 _DIAGNOSIS_IDS[status], 190 datestr, 191 url)) 192 193 194def _print_event_summary(event): 195 """Print a one-line summary of a job or special task.""" 196 start_time = time_utils.epoch_time_to_date_string( 197 event.start_time) 198 print(' %s %s %s' % ( 199 start_time, 200 _DIAGNOSIS_IDS[event.diagnosis], 201 event.job_url)) 202 203 204def _print_hosts(history_list, arguments): 205 """Print hosts, optionally with a job history. 206 207 This function handles both the default format for --working 208 and --broken options, as well as the output for the 209 --full_history and --diagnosis options. The `arguments` 210 parameter determines the format to use. 211 212 @param history_list A list of HostHistory objects to be printed. 213 @param arguments Parsed arguments object as returned by 214 ArgumentParser.parse_args(). 215 216 """ 217 for history in history_list: 218 status, _ = history.last_diagnosis() 219 if not _include_status(status, arguments): 220 continue 221 print(history.hostname) 222 if arguments.full_history: 223 for event in history: 224 _print_event_summary(event) 225 elif arguments.diagnosis: 226 for event in history.diagnosis_interval(): 227 _print_event_summary(event) 228 229 230def _validate_time_range(arguments): 231 """Validate the time range requested on the command line. 232 233 Enforces the rules for the --until, --since, and --duration 234 options are followed, and calculates defaults: 235 * It isn't allowed to supply all three options. 236 * If only two options are supplied, they completely determine 237 the time interval. 238 * If only one option is supplied, or no options, then apply 239 specified defaults to the arguments object. 240 241 @param arguments Parsed arguments object as returned by 242 ArgumentParser.parse_args(). 243 244 """ 245 if (arguments.duration is not None and 246 arguments.since is not None and arguments.until is not None): 247 print('FATAL: Can specify at most two of ' 248 '--since, --until, and --duration', 249 file=sys.stderr) 250 sys.exit(1) 251 if (arguments.until is None and (arguments.since is None or 252 arguments.duration is None)): 253 arguments.until = int(time.time()) 254 if arguments.since is None: 255 if arguments.duration is None: 256 arguments.duration = _DEFAULT_DURATION 257 arguments.since = (arguments.until - 258 arguments.duration * 60 * 60) 259 elif arguments.until is None: 260 arguments.until = (arguments.since + 261 arguments.duration * 60 * 60) 262 263 264def _get_host_histories(afe, arguments): 265 """Return HostJobHistory objects for the requested hosts. 266 267 Checks that individual hosts specified on the command line are 268 valid. Invalid hosts generate a warning message, and are 269 omitted from futher processing. 270 271 The return value is a list of HostJobHistory objects for the 272 valid requested hostnames, using the time range supplied on the 273 command line. 274 275 @param afe Autotest frontend 276 @param arguments Parsed arguments object as returned by 277 ArgumentParser.parse_args(). 278 @return List of HostJobHistory objects for the hosts requested 279 on the command line. 280 281 """ 282 histories = [] 283 saw_error = False 284 for hostname in arguments.hostnames: 285 try: 286 h = HostJobHistory.get_host_history( 287 afe, hostname, arguments.since, arguments.until) 288 histories.append(h) 289 except: 290 print('WARNING: Ignoring unknown host %s' % 291 hostname, file=sys.stderr) 292 saw_error = True 293 if saw_error: 294 # Create separation from the output that follows 295 print(file=sys.stderr) 296 return histories 297 298 299def _validate_host_list(afe, arguments): 300 """Validate the user-specified list of hosts. 301 302 Hosts may be specified implicitly with --board or --pool, or 303 explictly as command line arguments. This enforces these 304 rules: 305 * If --board or --pool, or both are specified, individual 306 hosts may not be specified. 307 * However specified, there must be at least one host. 308 309 The return value is a list of HostJobHistory objects for the 310 requested hosts, using the time range supplied on the command 311 line. 312 313 @param afe Autotest frontend 314 @param arguments Parsed arguments object as returned by 315 ArgumentParser.parse_args(). 316 @return List of HostJobHistory objects for the hosts requested 317 on the command line. 318 319 """ 320 if arguments.board or arguments.pool or arguments.model: 321 if arguments.hostnames: 322 print('FATAL: Hostname arguments provided ' 323 'with --board or --pool', file=sys.stderr) 324 sys.exit(1) 325 326 labels = labellib.LabelsMapping() 327 labels['board'] = arguments.board 328 labels['pool'] = arguments.pool 329 labels['model'] = arguments.model 330 histories = HostJobHistory.get_multiple_histories( 331 afe, arguments.since, arguments.until, labels.getlabels()) 332 else: 333 histories = _get_host_histories(afe, arguments) 334 if not histories: 335 print('FATAL: no valid hosts found', file=sys.stderr) 336 sys.exit(1) 337 return histories 338 339 340def _validate_format_options(arguments): 341 """Check the options for what output format to use. 342 343 Enforce these rules: 344 * If neither --broken nor --working was used, then --oneline 345 becomes the selected format. 346 * If neither --broken nor --working was used, included both 347 working and broken DUTs. 348 349 @param arguments Parsed arguments object as returned by 350 ArgumentParser.parse_args(). 351 352 """ 353 if (not arguments.oneline and not arguments.diagnosis and 354 not arguments.full_history): 355 arguments.oneline = (not arguments.working and 356 not arguments.broken) 357 if not arguments.working and not arguments.broken: 358 arguments.working = True 359 arguments.broken = True 360 361 362def _validate_command(afe, arguments): 363 """Check that the command's arguments are valid. 364 365 This performs command line checking to enforce command line 366 rules that ArgumentParser can't handle. Additionally, this 367 handles calculation of default arguments/options when a simple 368 constant default won't do. 369 370 Areas checked: 371 * Check that a valid time range was provided, supplying 372 defaults as necessary. 373 * Identify invalid host names. 374 375 @param afe Autotest frontend 376 @param arguments Parsed arguments object as returned by 377 ArgumentParser.parse_args(). 378 @return List of HostJobHistory objects for the hosts requested 379 on the command line. 380 381 """ 382 _validate_time_range(arguments) 383 _validate_format_options(arguments) 384 return _validate_host_list(afe, arguments) 385 386 387def _parse_command(argv): 388 """Parse the command line arguments. 389 390 Create an argument parser for this command's syntax, parse the 391 command line, and return the result of the ArgumentParser 392 parse_args() method. 393 394 @param argv Standard command line argument vector; argv[0] is 395 assumed to be the command name. 396 @return Result returned by ArgumentParser.parse_args(). 397 398 """ 399 parser = argparse.ArgumentParser( 400 prog=argv[0], 401 description='Report DUT status and execution history', 402 epilog='You can specify one or two of --since, --until, ' 403 'and --duration, but not all three.') 404 parser.add_argument('-s', '--since', type=status_history.parse_time, 405 metavar='DATE/TIME', 406 help=('Starting time for history display. ' 407 'Format: "YYYY-MM-DD HH:MM:SS"')) 408 parser.add_argument('-u', '--until', type=status_history.parse_time, 409 metavar='DATE/TIME', 410 help=('Ending time for history display. ' 411 'Format: "YYYY-MM-DD HH:MM:SS" ' 412 'Default: now')) 413 parser.add_argument('-d', '--duration', type=int, 414 metavar='HOURS', 415 help='Number of hours of history to display' 416 ' (default: %d)' % _DEFAULT_DURATION) 417 418 format_group = parser.add_mutually_exclusive_group() 419 format_group.add_argument('-f', '--full_history', action='store_true', 420 help='Display host history from most ' 421 'to least recent for each DUT') 422 format_group.add_argument('-g', '--diagnosis', action='store_true', 423 help='Display host history for the ' 424 'most recent DUT status change') 425 format_group.add_argument('-o', '--oneline', action='store_true', 426 help='Display host status summary') 427 428 parser.add_argument('-w', '--working', action='store_true', 429 help='List working devices by name only') 430 parser.add_argument('-n', '--broken', action='store_true', 431 help='List non-working devices by name only') 432 433 parser.add_argument('-b', '--board', 434 help='Display history for all DUTs ' 435 'of the given board') 436 parser.add_argument('-m', '--model', 437 help='Display history for all DUTs of the given model.') 438 parser.add_argument('-p', '--pool', 439 help='Display history for all DUTs ' 440 'in the given pool. You might ' 441 'be interested in the following pools: ' 442 + ', '.join(constants.Pools.MANAGED_POOLS[:-1]) 443 +', or '+ constants.Pools.MANAGED_POOLS[-1] +'.') 444 parser.add_argument('hostnames', 445 nargs='*', 446 help='Host names of DUTs to report on') 447 parser.add_argument('--web', 448 help='Autotest frontend hostname. If no value ' 449 'is given, the one in global config will be used.', 450 default=None) 451 arguments = parser.parse_args(argv[1:]) 452 return arguments 453 454 455def main(argv): 456 """Standard main() for command line processing. 457 458 @param argv Command line arguments (normally sys.argv). 459 460 """ 461 arguments = _parse_command(argv) 462 afe = frontend.AFE(server=arguments.web) 463 history_list = _validate_command(afe, arguments) 464 if arguments.oneline: 465 _print_host_summaries(history_list, arguments) 466 else: 467 _print_hosts(history_list, arguments) 468 469 470if __name__ == '__main__': 471 main(sys.argv) 472