1import os 2import random 3from subprocess import Popen, PIPE 4import psutil 5import json 6import sys 7import math 8import time 9from datetime import datetime 10 11# usage: python3 constantHelper.py JSON_FILE_PATH [BUILD_PATH] 12# 13# an example json config file is as follow: 14# visit https://bosc.yuque.com/yny0gi/gr7hyo/oy3dagqi9v97p696 for detail 15# { 16# "constants": [ 17# { 18# "name": "block_cycles_cache_0", 19# "width": 7, 20# "guide": 20, 21# "init": 11 22# }, 23# { 24# "name": "block_cycles_cache_1", 25# "width": 7, 26# "init": 18 27# }, 28# { 29# "name": "block_cycles_cache_2", 30# "width": 7, 31# "init": 127 32# }, 33# { 34# "name": "block_cycles_cache_3", 35# "width": 7, 36# "init": 17 37# } 38# ], 39# "opt_target": [ 40# {"successfully_forward_channel_D": {"policy" :"max", "baseline" :0} }, 41# {"successfully_forward_mshr": {"policy" :"max", "baseline" :0} }, 42# {"dcache.missQueue.entries_0: load_miss_penalty_to_use,": {"policy" :"min", "baseline" :250396} }, 43# {"dcache.missQueue.entries_1: load_miss_penalty_to_use,": {"policy" :"min", "baseline" :5634} }, 44# {"dcache.missQueue.entries_2: load_miss_penalty_to_use,": {"policy" :"min", "baseline" :4599} }, 45# {"dcache.missQueue.entries_3: load_miss_penalty_to_use,": {"policy" :"min", "baseline" :4146} } 46# ], 47 48# "population_num": 50, 49# "iteration_num": 50, 50# "crossover_rate": 50, 51# "mutation_rate": 50, 52 53# "emu_threads": 16, 54# "concurrent_emu": 4, 55# "max_instr": 1000000, 56# "seed": 3888, 57# "work_load": "~/nexus-am/apps/maprobe/build/maprobe-riscv64-xs.bin" 58# } 59 60 61# parameters according to noop 62NOOP_HOME = os.getenv("NOOP_HOME") 63XS_PROJECT_ROOT = os.getenv("XS_PROJECT_ROOT") 64if NOOP_HOME is None: 65 print("Please set NOOP_HOME first.") 66 exit(1) 67if XS_PROJECT_ROOT is None: 68 print("Please set XS_PROJECT_ROOT first.") 69 exit(1) 70DIFF_PATH = os.path.join(NOOP_HOME, "ready-to-run", "riscv64-nemu-interpreter-so") 71 72# get arguments 73if len(sys.argv) > 1: 74 JSON_FILE_PATH=sys.argv[1] 75else: 76 print("Please specify the json file path") 77 exit(1) 78if len(sys.argv) > 2: 79 BUILD_PATH = sys.argv[2] 80else: 81 BUILD_PATH = os.path.join(NOOP_HOME, "build") 82 83EMU_PATH = os.path.join(BUILD_PATH, "emu") 84CONFIG_FILE_PREFIX = ".constant_result_" 85PERF_FILE_POSTFIX = "tmp" 86MAXVAL = (1 << 63) - 1 87 88class Constant: 89 def __init__(self, obj: dict) -> None: 90 self.name = obj['name'] 91 self.width = obj['width'] 92 self.guide = (1 << self.width - 1) - 1 if 'guide' not in obj.keys() else obj['guide'] 93 self.init = random.randint(0, self.guide) if 'init' not in obj.keys() else obj['init'] 94 def maxrange(self) -> int: 95 return (1 << self.width) - 1 96 97 98class Config: 99 def __init__(self, constants, opt_target, population_num, iteration_num, crossover_rate, mutation_rate, emu_threads, concurrent_emu, max_instr, seed, work_load, tag) -> None: 100 self.constants = constants 101 self.opt_target = opt_target 102 self.population_num = int(population_num) 103 self.iteration_num = int(iteration_num) 104 self.crossover_rate = int(crossover_rate) 105 self.mutation_rate = int(mutation_rate) 106 self.emu_threads = int(emu_threads) 107 self.concurrent_emu = int(concurrent_emu) 108 self.max_instr = int(max_instr) 109 self.seed = int(seed) 110 self.work_load = work_load 111 self.tag = tag 112 def get_ith_constant(self, i) -> Constant: 113 return self.constants[i] 114 def get_constain_num(self) -> int: 115 return len(self.constants) 116 117 118def loadConfig(json_path, tag) -> Config: 119 obj = json.load(open(json_path, "r")) 120 constants = [Constant(obj['constants'][i]) for i in range(len(obj['constants']))] 121 config = Config(constants, obj['opt_target'], obj['population_num'], obj['iteration_num'], obj['crossover_rate'], obj['mutation_rate'], obj['emu_threads'], obj['concurrent_emu'], obj['max_instr'], obj['seed'], obj['work_load'], tag) 122 return config 123 124class RunContext: 125 def __init__(self, config: Config) -> None: 126 self.config = config 127 def checkCoreFree(self) -> bool: 128 percent_per_core = psutil.cpu_percent(interval=1 ,percpu=True) 129 acc = 0 130 for i in range(self.config.concurrent_emu * self.config.emu_threads): 131 acc += percent_per_core[i] 132 if acc < (0.1 * (100 * self.config.concurrent_emu * self.config.emu_threads)): 133 return True 134 else: 135 print("no free {} core, core usage:".format(self.config.concurrent_emu * self.config.emu_threads)) 136 print(percent_per_core) 137 return False 138 def get_free_cores(self) -> tuple[bool, int, int, int]: 139 thread = self.config.emu_threads 140 # return (Success?, numa node, start_core, end_core) 141 num_core = psutil.cpu_count(logical=False) # SMT is not allowed 142 core_usage = psutil.cpu_percent(interval=2, percpu=True) 143 num_window = num_core // thread 144 for i in range(num_window): 145 start = i * thread 146 end = (i + 1) * thread 147 window_usage = core_usage[start:end] 148 free = sum(window_usage) < 30 * thread and True not in map(lambda x: x > 80, window_usage) 149 if free: 150 return (True, int(start >= (num_core // 2)), start, end - 1) 151 return (False, 0, 0, 0) 152 def getStdIn(self, population: list, id: int) -> str: 153 res = 'echo \"' 154 res += str(len(population[id])) 155 res += '\\n' 156 for item in population[id]: 157 res += item[0] + ' ' + str(item[1]) + '\\n' 158 res += '\"' 159 return res 160 161 def genRunCMD(self, population, id, numa = None, coreStart = None, coreEnd = None) -> str: 162 stdinStr = self.getStdIn(population, id) 163 if None in [numa, coreStart, coreEnd]: 164 return "{} | {} -i {} --diff {} -I {} -s {}".format(stdinStr, EMU_PATH, self.config.work_load, DIFF_PATH, self.config.max_instr, self.config.seed) 165 return "{} | numactl -m {} -C {}-{} {} -i {} --diff {} -I {} -s {}".format(stdinStr, numa, coreStart, coreEnd, EMU_PATH, self.config.work_load, DIFF_PATH, self.config.max_instr, self.config.seed) 166 167 def getOutPath(self, iterid, i): 168 dirPath = os.path.join(BUILD_PATH, self.config.tag) 169 if not os.path.exists(dirPath): 170 os.mkdir(dirPath) 171 return os.path.join(dirPath, f"{iterid}-{i}-out.txt") 172 173 def getPerfPath(self, iterid, i): 174 # return os.path.join(BUILD_PATH, CONFIG_FILE_PREFIX + str(i) + '.' + PERF_FILE_POSTFIX) 175 dirPath = os.path.join(BUILD_PATH, self.config.tag) 176 if not os.path.exists(dirPath): 177 os.mkdir(dirPath) 178 return os.path.join(dirPath, f"{iterid}-{i}-err.txt") 179 180class Solution: 181 def __init__(self, config: Config) -> None: 182 self.config = config 183 self.context = RunContext(config) 184 def genFirstPopulation(self) -> list: 185 res = [] 186 used = [] 187 config = self.config 188 for i in range(config.population_num): 189 candidate = [[config.get_ith_constant(i).name, random.randint(0, config.get_ith_constant(i).maxrange()) % config.get_ith_constant(i).guide] for i in range(config.get_constain_num())] 190 while(candidate in used): 191 candidate = [[config.get_ith_constant(i).name, random.randint(0, config.get_ith_constant(i).maxrange()) % config.get_ith_constant(i).guide] for i in range(config.get_constain_num())] 192 used.append(candidate) 193 res.append(candidate) 194 assert(len(res) == config.population_num) 195 return res 196 def profilling_fitness(self, iterid: int) -> list: 197 fitness = [] 198 lines = [] 199 for idx in range(self.config.population_num): 200 with open(self.context.getPerfPath(iterid, idx), "r") as fp: 201 lines = fp.readlines() 202 res = 0 203 for line in lines: 204 for opt in config.opt_target: 205 if list(opt.keys())[0] in line: 206 # max and min policy 207 if list(opt.values())[0]['policy'] == 'max': 208 res += int(list(filter(lambda x: x != '', line.split(' ')))[-1]) - int(list(opt.values())[0]['baseline']) 209 elif list(opt.values())[0]['policy'] == 'min': 210 res += int(list(opt.values())[0]['baseline']) - int(list(filter(lambda x: x != '', line.split(' ')))[-1]) 211 fitness.append(res) 212 assert(len(fitness) == self.config.population_num) 213 return fitness 214 def run_one_round(self, iterid: int, population: list) -> None: 215 procs = [] 216 i = 0 217 while i < len(population): 218 if i % self.config.concurrent_emu == 0: 219 for proc in procs: 220 proc.wait() 221 procs.clear() 222 print(population[i]) 223 # while True: 224 # (succ, numa, coreStart, coreEnd) = self.context.get_free_cores() 225 # if succ: 226 # with open(self.context.getOutPath(iterid, i), "w") as stdout, open(self.context.getPerfPath(iterid, i), "w") as stderr: 227 # # print(self.context.genRunCMD(population, i, numa, coreStart, coreEnd), flush=True) 228 # procs.append(Popen(args=self.context.genRunCMD(population, i, numa, coreStart, coreEnd), shell=True, encoding='utf-8', stdin=PIPE, stdout=stdout, stderr=stderr)) 229 # break 230 # print("no free {} core".format(self.config.concurrent_emu * self.config.emu_threads)) 231 # time.sleep(5) 232 ## only for tutorial 233 with open(self.context.getOutPath(iterid, i), "w") as stdout, open(self.context.getPerfPath(iterid, i), "w") as stderr: 234 procs.append(Popen(args=self.context.genRunCMD(population, i), shell=True, encoding='utf-8', stdin=PIPE, stdout=stdout, stderr=stderr)) 235 i += 1 236 for proc in procs: 237 proc.wait() 238 def mutation(self, item: list) -> list: 239 res = [] 240 for val in item: 241 width = 0 242 guide = 0 243 for constant in self.config.constants: 244 if(constant.name == val[0]): 245 width = constant.width 246 guide = constant.guide 247 mask = 1 << random.randint(0, width - 1) 248 if random.randint(0, 100) > self.config.mutation_rate: 249 res.append(val) 250 else: 251 val[1] = (((val[1] & mask) ^ mask) | val[1]) % guide 252 res.append(val) 253 assert(len(item) == len(res)) 254 return res 255 def crossover(self, poplulation: list) -> list: 256 res = [] 257 if len(poplulation) < 2: 258 return poplulation 259 for individual in poplulation: 260 indivi = [] 261 for (index, constant) in enumerate(individual): 262 const = constant 263 if random.randint(0, 100) < self.config.crossover_rate: 264 crossover_target_id = 0 265 while crossover_target_id == index: 266 crossover_target_id = random.randint(0, len(poplulation) - 1) 267 maskMax = 0 268 guide = 0 269 for config_const in self.config.constants: 270 if config_const.name == constant[0]: 271 maskMax = config_const.width 272 guide = config_const.guide 273 maskMax = int(math.log2(guide)) + 1 if (int(math.log2(guide)) + 1 < maskMax) else maskMax 274 maskLen = random.randint(1, maskMax) 275 mask = (1 << maskLen) - 1 276 shiftLen = random.randint(0, maskMax - maskLen) 277 mask = mask << shiftLen 278 const_now = const[1] 279 target_now = poplulation[crossover_target_id][index][1] 280 const_now = ((const_now & ~(mask)) | (target_now & mask)) % guide 281 const = [constant[0], const_now] 282 indivi.append(const) 283 res.append(indivi) 284 assert(len(poplulation) == len(res)) 285 return res 286 def genNextPop(self, curPop, fitness) -> list: 287 nextgen = [] 288 tmp = sorted(zip(curPop, fitness), key=lambda x : x[1], reverse=True) 289 print() 290 print("opt constant in this round is ", list(tmp)[0][0], " fitness is ", int(list(tmp)[0][1])) 291 cross = [] 292 for i in range(len(tmp)): 293 if i < (len(tmp) // 2): 294 # select 295 nextgen.append(tmp[i][0]) 296 else: 297 cross.append(tmp[i][0]) 298 # crossover 299 cross = self.crossover(cross) 300 nextgen = nextgen + cross 301 # mutation 302 for i in range(len(tmp)): 303 nextgen[i] = self.mutation(nextgen[i]) 304 assert(len(curPop) == len(nextgen)) 305 return nextgen 306 307 class HashList: 308 def __init__(self, obj: list) -> None: 309 # obj: [['test1', 38], ['test2', 15]] 310 self.obj = obj 311 def __hash__(self) -> str: 312 res = '' 313 for const in self.obj: 314 res += ' '.join(map(lambda x : str(x), const)) 315 return hash(res) 316 def __eq__(self, __o: object) -> bool: 317 for (idx, const) in enumerate(self.obj): 318 if const != __o.obj[idx]: 319 return False 320 return True 321 322 def gene_cal(self) -> None: 323 globalMap = dict() 324 if(self.config.population_num % 2 != 0): 325 print("gene algrithom must ensure that population_num is an even value") 326 return 327 parentPoplation = self.genFirstPopulation() 328 init_indiv = [] 329 for constant in self.config.constants: 330 const = [] 331 const.append(constant.name) 332 const.append(constant.init) 333 init_indiv.append(const) 334 parentPoplation.pop() 335 parentPoplation.append(init_indiv) 336 for i in range(self.config.iteration_num): 337 if i != 0: 338 print() 339 print("iteration ", i, " begins") 340 print() 341 self.run_one_round(i, parentPoplation) 342 fitness = self.profilling_fitness(i) 343 for (pop, fit) in zip(parentPoplation, fitness): 344 globalMap[self.HashList(pop)] = fit 345 parentPoplation = self.genNextPop(parentPoplation, fitness) 346 347 globalMap = zip(globalMap.keys(), globalMap.values()) 348 globalMap = sorted(globalMap, key=lambda x : x[1], reverse=True) 349 print("opt constant for gene algrithom is ", list(globalMap)[0][0].obj, " fitness", int(list(globalMap)[0][1])) 350 351tid = datetime.now().strftime("%m%d%H%M") 352config = loadConfig(JSON_FILE_PATH, f"constantin_{tid}") 353Solution(config).gene_cal() 354