1*760c253cSXin Li# Copyright 2013 The ChromiumOS Authors 2*760c253cSXin Li# Use of this source code is governed by a BSD-style license that can be 3*760c253cSXin Li# found in the LICENSE file. 4*760c253cSXin Li"""An example main file running the algorithms. 5*760c253cSXin Li 6*760c253cSXin LiPart of the Chrome build flags optimization. 7*760c253cSXin Li 8*760c253cSXin LiAn example use of the framework. It parses the input json configuration file. 9*760c253cSXin LiThen it initiates the variables of the generation. Finally, it sets up the 10*760c253cSXin Liprocesses for different modules and runs the experiment. 11*760c253cSXin Li""" 12*760c253cSXin Li 13*760c253cSXin Li__author__ = "[email protected] (Yuheng Long)" 14*760c253cSXin Li 15*760c253cSXin Liimport json 16*760c253cSXin Liimport multiprocessing 17*760c253cSXin Lifrom optparse import OptionParser 18*760c253cSXin Liimport sys 19*760c253cSXin Li 20*760c253cSXin Liimport flags 21*760c253cSXin Lifrom genetic_algorithm import GAGeneration 22*760c253cSXin Lifrom pipeline_process import PipelineProcess 23*760c253cSXin Liimport pipeline_worker 24*760c253cSXin Lifrom steering import Steering 25*760c253cSXin Lifrom task import BUILD_STAGE 26*760c253cSXin Lifrom task import Task 27*760c253cSXin Lifrom task import TEST_STAGE 28*760c253cSXin Liimport testing_batch 29*760c253cSXin Li 30*760c253cSXin Li 31*760c253cSXin Liparser = OptionParser() 32*760c253cSXin Li 33*760c253cSXin Liparser.add_option( 34*760c253cSXin Li "-f", 35*760c253cSXin Li "--file", 36*760c253cSXin Li dest="filename", 37*760c253cSXin Li help="configuration file FILE input", 38*760c253cSXin Li metavar="FILE", 39*760c253cSXin Li) 40*760c253cSXin Li 41*760c253cSXin Li# The meta data for the genetic algorithm. 42*760c253cSXin LiBUILD_CMD = "BUILD_CMD" 43*760c253cSXin LiTEST_CMD = "TEST_CMD" 44*760c253cSXin LiOUTPUT = "OUTPUT" 45*760c253cSXin LiDEFAULT_OUTPUT = "output" 46*760c253cSXin LiCONF = "CONF" 47*760c253cSXin LiDEFAULT_CONF = "conf" 48*760c253cSXin LiNUM_BUILDER = "NUM_BUILDER" 49*760c253cSXin LiDEFAULT_NUM_BUILDER = 1 50*760c253cSXin LiNUM_TESTER = "NUM_TESTER" 51*760c253cSXin LiDEFAULT_NUM_TESTER = 1 52*760c253cSXin LiSTOP_THRESHOLD = "STOP_THRESHOLD" 53*760c253cSXin LiDEFAULT_STOP_THRESHOLD = 1 54*760c253cSXin LiNUM_CHROMOSOMES = "NUM_CHROMOSOMES" 55*760c253cSXin LiDEFAULT_NUM_CHROMOSOMES = 20 56*760c253cSXin LiNUM_TRIALS = "NUM_TRIALS" 57*760c253cSXin LiDEFAULT_NUM_TRIALS = 20 58*760c253cSXin LiMUTATION_RATE = "MUTATION_RATE" 59*760c253cSXin LiDEFAULT_MUTATION_RATE = 0.01 60*760c253cSXin Li 61*760c253cSXin Li 62*760c253cSXin Lidef _ProcessGA(meta_data): 63*760c253cSXin Li """Set up the meta data for the genetic algorithm. 64*760c253cSXin Li 65*760c253cSXin Li Args: 66*760c253cSXin Li meta_data: the meta data for the genetic algorithm. 67*760c253cSXin Li """ 68*760c253cSXin Li assert BUILD_CMD in meta_data 69*760c253cSXin Li build_cmd = meta_data[BUILD_CMD] 70*760c253cSXin Li 71*760c253cSXin Li assert TEST_CMD in meta_data 72*760c253cSXin Li test_cmd = meta_data[TEST_CMD] 73*760c253cSXin Li 74*760c253cSXin Li if OUTPUT not in meta_data: 75*760c253cSXin Li output_file = DEFAULT_OUTPUT 76*760c253cSXin Li else: 77*760c253cSXin Li output_file = meta_data[OUTPUT] 78*760c253cSXin Li 79*760c253cSXin Li if CONF not in meta_data: 80*760c253cSXin Li conf_file = DEFAULT_CONF 81*760c253cSXin Li else: 82*760c253cSXin Li conf_file = meta_data[CONF] 83*760c253cSXin Li 84*760c253cSXin Li if NUM_BUILDER not in meta_data: 85*760c253cSXin Li num_builders = DEFAULT_NUM_BUILDER 86*760c253cSXin Li else: 87*760c253cSXin Li num_builders = meta_data[NUM_BUILDER] 88*760c253cSXin Li 89*760c253cSXin Li if NUM_TESTER not in meta_data: 90*760c253cSXin Li num_testers = DEFAULT_NUM_TESTER 91*760c253cSXin Li else: 92*760c253cSXin Li num_testers = meta_data[NUM_TESTER] 93*760c253cSXin Li 94*760c253cSXin Li if STOP_THRESHOLD not in meta_data: 95*760c253cSXin Li stop_threshold = DEFAULT_STOP_THRESHOLD 96*760c253cSXin Li else: 97*760c253cSXin Li stop_threshold = meta_data[STOP_THRESHOLD] 98*760c253cSXin Li 99*760c253cSXin Li if NUM_CHROMOSOMES not in meta_data: 100*760c253cSXin Li num_chromosomes = DEFAULT_NUM_CHROMOSOMES 101*760c253cSXin Li else: 102*760c253cSXin Li num_chromosomes = meta_data[NUM_CHROMOSOMES] 103*760c253cSXin Li 104*760c253cSXin Li if NUM_TRIALS not in meta_data: 105*760c253cSXin Li num_trials = DEFAULT_NUM_TRIALS 106*760c253cSXin Li else: 107*760c253cSXin Li num_trials = meta_data[NUM_TRIALS] 108*760c253cSXin Li 109*760c253cSXin Li if MUTATION_RATE not in meta_data: 110*760c253cSXin Li mutation_rate = DEFAULT_MUTATION_RATE 111*760c253cSXin Li else: 112*760c253cSXin Li mutation_rate = meta_data[MUTATION_RATE] 113*760c253cSXin Li 114*760c253cSXin Li specs = flags.ReadConf(conf_file) 115*760c253cSXin Li 116*760c253cSXin Li # Initiate the build/test command and the log directory. 117*760c253cSXin Li Task.InitLogCommand(build_cmd, test_cmd, output_file) 118*760c253cSXin Li 119*760c253cSXin Li # Initiate the build/test command and the log directory. 120*760c253cSXin Li GAGeneration.InitMetaData( 121*760c253cSXin Li stop_threshold, num_chromosomes, num_trials, specs, mutation_rate 122*760c253cSXin Li ) 123*760c253cSXin Li 124*760c253cSXin Li # Generate the initial generations. 125*760c253cSXin Li generation_tasks = testing_batch.GenerateRandomGATasks( 126*760c253cSXin Li specs, num_chromosomes, num_trials 127*760c253cSXin Li ) 128*760c253cSXin Li generations = [GAGeneration(generation_tasks, set([]), 0)] 129*760c253cSXin Li 130*760c253cSXin Li # Execute the experiment. 131*760c253cSXin Li _StartExperiment(num_builders, num_testers, generations) 132*760c253cSXin Li 133*760c253cSXin Li 134*760c253cSXin Lidef _ParseJson(file_name): 135*760c253cSXin Li """Parse the input json file. 136*760c253cSXin Li 137*760c253cSXin Li Parse the input json file and call the proper function to perform the 138*760c253cSXin Li algorithms. 139*760c253cSXin Li 140*760c253cSXin Li Args: 141*760c253cSXin Li file_name: the input json file name. 142*760c253cSXin Li """ 143*760c253cSXin Li 144*760c253cSXin Li experiments = json.load(open(file_name)) 145*760c253cSXin Li 146*760c253cSXin Li for experiment in experiments: 147*760c253cSXin Li if experiment == "GA": 148*760c253cSXin Li # An GA experiment 149*760c253cSXin Li _ProcessGA(experiments[experiment]) 150*760c253cSXin Li 151*760c253cSXin Li 152*760c253cSXin Lidef _StartExperiment(num_builders, num_testers, generations): 153*760c253cSXin Li """Set up the experiment environment and execute the framework. 154*760c253cSXin Li 155*760c253cSXin Li Args: 156*760c253cSXin Li num_builders: number of concurrent builders. 157*760c253cSXin Li num_testers: number of concurrent testers. 158*760c253cSXin Li generations: the initial generation for the framework. 159*760c253cSXin Li """ 160*760c253cSXin Li 161*760c253cSXin Li manager = multiprocessing.Manager() 162*760c253cSXin Li 163*760c253cSXin Li # The queue between the steering algorithm and the builder. 164*760c253cSXin Li steering_build = manager.Queue() 165*760c253cSXin Li # The queue between the builder and the tester. 166*760c253cSXin Li build_test = manager.Queue() 167*760c253cSXin Li # The queue between the tester and the steering algorithm. 168*760c253cSXin Li test_steering = manager.Queue() 169*760c253cSXin Li 170*760c253cSXin Li # Set up the processes for the builder, tester and steering algorithm module. 171*760c253cSXin Li build_process = PipelineProcess( 172*760c253cSXin Li num_builders, 173*760c253cSXin Li "builder", 174*760c253cSXin Li {}, 175*760c253cSXin Li BUILD_STAGE, 176*760c253cSXin Li steering_build, 177*760c253cSXin Li pipeline_worker.Helper, 178*760c253cSXin Li pipeline_worker.Worker, 179*760c253cSXin Li build_test, 180*760c253cSXin Li ) 181*760c253cSXin Li 182*760c253cSXin Li test_process = PipelineProcess( 183*760c253cSXin Li num_testers, 184*760c253cSXin Li "tester", 185*760c253cSXin Li {}, 186*760c253cSXin Li TEST_STAGE, 187*760c253cSXin Li build_test, 188*760c253cSXin Li pipeline_worker.Helper, 189*760c253cSXin Li pipeline_worker.Worker, 190*760c253cSXin Li test_steering, 191*760c253cSXin Li ) 192*760c253cSXin Li 193*760c253cSXin Li steer_process = multiprocessing.Process( 194*760c253cSXin Li target=Steering, 195*760c253cSXin Li args=(set([]), generations, test_steering, steering_build), 196*760c253cSXin Li ) 197*760c253cSXin Li 198*760c253cSXin Li # Start the processes. 199*760c253cSXin Li build_process.start() 200*760c253cSXin Li test_process.start() 201*760c253cSXin Li steer_process.start() 202*760c253cSXin Li 203*760c253cSXin Li # Wait for the processes to finish. 204*760c253cSXin Li build_process.join() 205*760c253cSXin Li test_process.join() 206*760c253cSXin Li steer_process.join() 207*760c253cSXin Li 208*760c253cSXin Li 209*760c253cSXin Lidef main(argv): 210*760c253cSXin Li (options, _) = parser.parse_args(argv) 211*760c253cSXin Li assert options.filename 212*760c253cSXin Li _ParseJson(options.filename) 213*760c253cSXin Li 214*760c253cSXin Li 215*760c253cSXin Liif __name__ == "__main__": 216*760c253cSXin Li main(sys.argv) 217