xref: /aosp_15_r20/external/toolchain-utils/bestflags/example_algorithms.py (revision 760c253c1ed00ce9abd48f8546f08516e57485fe)
1*760c253cSXin Li# Copyright 2013 The ChromiumOS Authors
2*760c253cSXin Li# Use of this source code is governed by a BSD-style license that can be
3*760c253cSXin Li# found in the LICENSE file.
4*760c253cSXin Li"""An example main file running the algorithms.
5*760c253cSXin Li
6*760c253cSXin LiPart of the Chrome build flags optimization.
7*760c253cSXin Li
8*760c253cSXin LiAn example use of the framework. It parses the input json configuration file.
9*760c253cSXin LiThen it initiates the variables of the generation. Finally, it sets up the
10*760c253cSXin Liprocesses for different modules and runs the experiment.
11*760c253cSXin Li"""
12*760c253cSXin Li
13*760c253cSXin Li__author__ = "[email protected] (Yuheng Long)"
14*760c253cSXin Li
15*760c253cSXin Liimport json
16*760c253cSXin Liimport multiprocessing
17*760c253cSXin Lifrom optparse import OptionParser
18*760c253cSXin Liimport sys
19*760c253cSXin Li
20*760c253cSXin Liimport flags
21*760c253cSXin Lifrom genetic_algorithm import GAGeneration
22*760c253cSXin Lifrom pipeline_process import PipelineProcess
23*760c253cSXin Liimport pipeline_worker
24*760c253cSXin Lifrom steering import Steering
25*760c253cSXin Lifrom task import BUILD_STAGE
26*760c253cSXin Lifrom task import Task
27*760c253cSXin Lifrom task import TEST_STAGE
28*760c253cSXin Liimport testing_batch
29*760c253cSXin Li
30*760c253cSXin Li
31*760c253cSXin Liparser = OptionParser()
32*760c253cSXin Li
33*760c253cSXin Liparser.add_option(
34*760c253cSXin Li    "-f",
35*760c253cSXin Li    "--file",
36*760c253cSXin Li    dest="filename",
37*760c253cSXin Li    help="configuration file FILE input",
38*760c253cSXin Li    metavar="FILE",
39*760c253cSXin Li)
40*760c253cSXin Li
41*760c253cSXin Li# The meta data for the genetic algorithm.
42*760c253cSXin LiBUILD_CMD = "BUILD_CMD"
43*760c253cSXin LiTEST_CMD = "TEST_CMD"
44*760c253cSXin LiOUTPUT = "OUTPUT"
45*760c253cSXin LiDEFAULT_OUTPUT = "output"
46*760c253cSXin LiCONF = "CONF"
47*760c253cSXin LiDEFAULT_CONF = "conf"
48*760c253cSXin LiNUM_BUILDER = "NUM_BUILDER"
49*760c253cSXin LiDEFAULT_NUM_BUILDER = 1
50*760c253cSXin LiNUM_TESTER = "NUM_TESTER"
51*760c253cSXin LiDEFAULT_NUM_TESTER = 1
52*760c253cSXin LiSTOP_THRESHOLD = "STOP_THRESHOLD"
53*760c253cSXin LiDEFAULT_STOP_THRESHOLD = 1
54*760c253cSXin LiNUM_CHROMOSOMES = "NUM_CHROMOSOMES"
55*760c253cSXin LiDEFAULT_NUM_CHROMOSOMES = 20
56*760c253cSXin LiNUM_TRIALS = "NUM_TRIALS"
57*760c253cSXin LiDEFAULT_NUM_TRIALS = 20
58*760c253cSXin LiMUTATION_RATE = "MUTATION_RATE"
59*760c253cSXin LiDEFAULT_MUTATION_RATE = 0.01
60*760c253cSXin Li
61*760c253cSXin Li
62*760c253cSXin Lidef _ProcessGA(meta_data):
63*760c253cSXin Li    """Set up the meta data for the genetic algorithm.
64*760c253cSXin Li
65*760c253cSXin Li    Args:
66*760c253cSXin Li      meta_data: the meta data for the genetic algorithm.
67*760c253cSXin Li    """
68*760c253cSXin Li    assert BUILD_CMD in meta_data
69*760c253cSXin Li    build_cmd = meta_data[BUILD_CMD]
70*760c253cSXin Li
71*760c253cSXin Li    assert TEST_CMD in meta_data
72*760c253cSXin Li    test_cmd = meta_data[TEST_CMD]
73*760c253cSXin Li
74*760c253cSXin Li    if OUTPUT not in meta_data:
75*760c253cSXin Li        output_file = DEFAULT_OUTPUT
76*760c253cSXin Li    else:
77*760c253cSXin Li        output_file = meta_data[OUTPUT]
78*760c253cSXin Li
79*760c253cSXin Li    if CONF not in meta_data:
80*760c253cSXin Li        conf_file = DEFAULT_CONF
81*760c253cSXin Li    else:
82*760c253cSXin Li        conf_file = meta_data[CONF]
83*760c253cSXin Li
84*760c253cSXin Li    if NUM_BUILDER not in meta_data:
85*760c253cSXin Li        num_builders = DEFAULT_NUM_BUILDER
86*760c253cSXin Li    else:
87*760c253cSXin Li        num_builders = meta_data[NUM_BUILDER]
88*760c253cSXin Li
89*760c253cSXin Li    if NUM_TESTER not in meta_data:
90*760c253cSXin Li        num_testers = DEFAULT_NUM_TESTER
91*760c253cSXin Li    else:
92*760c253cSXin Li        num_testers = meta_data[NUM_TESTER]
93*760c253cSXin Li
94*760c253cSXin Li    if STOP_THRESHOLD not in meta_data:
95*760c253cSXin Li        stop_threshold = DEFAULT_STOP_THRESHOLD
96*760c253cSXin Li    else:
97*760c253cSXin Li        stop_threshold = meta_data[STOP_THRESHOLD]
98*760c253cSXin Li
99*760c253cSXin Li    if NUM_CHROMOSOMES not in meta_data:
100*760c253cSXin Li        num_chromosomes = DEFAULT_NUM_CHROMOSOMES
101*760c253cSXin Li    else:
102*760c253cSXin Li        num_chromosomes = meta_data[NUM_CHROMOSOMES]
103*760c253cSXin Li
104*760c253cSXin Li    if NUM_TRIALS not in meta_data:
105*760c253cSXin Li        num_trials = DEFAULT_NUM_TRIALS
106*760c253cSXin Li    else:
107*760c253cSXin Li        num_trials = meta_data[NUM_TRIALS]
108*760c253cSXin Li
109*760c253cSXin Li    if MUTATION_RATE not in meta_data:
110*760c253cSXin Li        mutation_rate = DEFAULT_MUTATION_RATE
111*760c253cSXin Li    else:
112*760c253cSXin Li        mutation_rate = meta_data[MUTATION_RATE]
113*760c253cSXin Li
114*760c253cSXin Li    specs = flags.ReadConf(conf_file)
115*760c253cSXin Li
116*760c253cSXin Li    # Initiate the build/test command and the log directory.
117*760c253cSXin Li    Task.InitLogCommand(build_cmd, test_cmd, output_file)
118*760c253cSXin Li
119*760c253cSXin Li    # Initiate the build/test command and the log directory.
120*760c253cSXin Li    GAGeneration.InitMetaData(
121*760c253cSXin Li        stop_threshold, num_chromosomes, num_trials, specs, mutation_rate
122*760c253cSXin Li    )
123*760c253cSXin Li
124*760c253cSXin Li    # Generate the initial generations.
125*760c253cSXin Li    generation_tasks = testing_batch.GenerateRandomGATasks(
126*760c253cSXin Li        specs, num_chromosomes, num_trials
127*760c253cSXin Li    )
128*760c253cSXin Li    generations = [GAGeneration(generation_tasks, set([]), 0)]
129*760c253cSXin Li
130*760c253cSXin Li    # Execute the experiment.
131*760c253cSXin Li    _StartExperiment(num_builders, num_testers, generations)
132*760c253cSXin Li
133*760c253cSXin Li
134*760c253cSXin Lidef _ParseJson(file_name):
135*760c253cSXin Li    """Parse the input json file.
136*760c253cSXin Li
137*760c253cSXin Li    Parse the input json file and call the proper function to perform the
138*760c253cSXin Li    algorithms.
139*760c253cSXin Li
140*760c253cSXin Li    Args:
141*760c253cSXin Li      file_name: the input json file name.
142*760c253cSXin Li    """
143*760c253cSXin Li
144*760c253cSXin Li    experiments = json.load(open(file_name))
145*760c253cSXin Li
146*760c253cSXin Li    for experiment in experiments:
147*760c253cSXin Li        if experiment == "GA":
148*760c253cSXin Li            # An GA experiment
149*760c253cSXin Li            _ProcessGA(experiments[experiment])
150*760c253cSXin Li
151*760c253cSXin Li
152*760c253cSXin Lidef _StartExperiment(num_builders, num_testers, generations):
153*760c253cSXin Li    """Set up the experiment environment and execute the framework.
154*760c253cSXin Li
155*760c253cSXin Li    Args:
156*760c253cSXin Li      num_builders: number of concurrent builders.
157*760c253cSXin Li      num_testers: number of concurrent testers.
158*760c253cSXin Li      generations: the initial generation for the framework.
159*760c253cSXin Li    """
160*760c253cSXin Li
161*760c253cSXin Li    manager = multiprocessing.Manager()
162*760c253cSXin Li
163*760c253cSXin Li    # The queue between the steering algorithm and the builder.
164*760c253cSXin Li    steering_build = manager.Queue()
165*760c253cSXin Li    # The queue between the builder and the tester.
166*760c253cSXin Li    build_test = manager.Queue()
167*760c253cSXin Li    # The queue between the tester and the steering algorithm.
168*760c253cSXin Li    test_steering = manager.Queue()
169*760c253cSXin Li
170*760c253cSXin Li    # Set up the processes for the builder, tester and steering algorithm module.
171*760c253cSXin Li    build_process = PipelineProcess(
172*760c253cSXin Li        num_builders,
173*760c253cSXin Li        "builder",
174*760c253cSXin Li        {},
175*760c253cSXin Li        BUILD_STAGE,
176*760c253cSXin Li        steering_build,
177*760c253cSXin Li        pipeline_worker.Helper,
178*760c253cSXin Li        pipeline_worker.Worker,
179*760c253cSXin Li        build_test,
180*760c253cSXin Li    )
181*760c253cSXin Li
182*760c253cSXin Li    test_process = PipelineProcess(
183*760c253cSXin Li        num_testers,
184*760c253cSXin Li        "tester",
185*760c253cSXin Li        {},
186*760c253cSXin Li        TEST_STAGE,
187*760c253cSXin Li        build_test,
188*760c253cSXin Li        pipeline_worker.Helper,
189*760c253cSXin Li        pipeline_worker.Worker,
190*760c253cSXin Li        test_steering,
191*760c253cSXin Li    )
192*760c253cSXin Li
193*760c253cSXin Li    steer_process = multiprocessing.Process(
194*760c253cSXin Li        target=Steering,
195*760c253cSXin Li        args=(set([]), generations, test_steering, steering_build),
196*760c253cSXin Li    )
197*760c253cSXin Li
198*760c253cSXin Li    # Start the processes.
199*760c253cSXin Li    build_process.start()
200*760c253cSXin Li    test_process.start()
201*760c253cSXin Li    steer_process.start()
202*760c253cSXin Li
203*760c253cSXin Li    # Wait for the processes to finish.
204*760c253cSXin Li    build_process.join()
205*760c253cSXin Li    test_process.join()
206*760c253cSXin Li    steer_process.join()
207*760c253cSXin Li
208*760c253cSXin Li
209*760c253cSXin Lidef main(argv):
210*760c253cSXin Li    (options, _) = parser.parse_args(argv)
211*760c253cSXin Li    assert options.filename
212*760c253cSXin Li    _ParseJson(options.filename)
213*760c253cSXin Li
214*760c253cSXin Li
215*760c253cSXin Liif __name__ == "__main__":
216*760c253cSXin Li    main(sys.argv)
217