1#*************************************************************************************** 2# Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3# Copyright (c) 2020-2021 Peng Cheng Laboratory 4# 5# XiangShan is licensed under Mulan PSL v2. 6# You can use this software according to the terms and conditions of the Mulan PSL v2. 7# You may obtain a copy of Mulan PSL v2 at: 8# http://license.coscl.org.cn/MulanPSL2 9# 10# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13# 14# See the Mulan PSL v2 for more details. 15#*************************************************************************************** 16 17# Simple version of xiangshan python wrapper 18 19import argparse 20import json 21import os 22import random 23import signal 24import subprocess 25import sys 26import time 27 28import psutil 29 30 31def load_all_gcpt(gcpt_path, json_path): 32 all_gcpt = [] 33 with open(json_path) as f: 34 data = json.load(f) 35 for benchspec in data: 36 for point in data[benchspec]: 37 weight = data[benchspec][point] 38 gcpt = os.path.join(gcpt_path, "_".join([benchspec, point, weight])) 39 bin_dir = os.path.join(gcpt, "0") 40 bin_file = list(os.listdir(bin_dir)) 41 assert(len(bin_file) == 1) 42 bin_path = os.path.join(bin_dir, bin_file[0]) 43 assert(os.path.isfile(bin_path)) 44 all_gcpt.append(bin_path) 45 return all_gcpt 46 47class XSArgs(object): 48 script_path = os.path.realpath(__file__) 49 # default path to the repositories 50 noop_home = os.path.join(os.path.dirname(script_path), "..") 51 nemu_home = os.path.join(noop_home, "../NEMU") 52 am_home = os.path.join(noop_home, "../nexus-am") 53 dramsim3_home = os.path.join(noop_home, "../DRAMsim3") 54 rvtest_home = os.path.join(noop_home, "../riscv-tests") 55 default_wave_home = os.path.join(noop_home, "build") 56 wave_home = default_wave_home 57 58 def __init__(self, args): 59 # all path environment variables that should be set 60 all_path = [ 61 # (python argument, environment variable, default, target function) 62 (None, "NOOP_HOME", self.noop_home, self.set_noop_home), 63 (args.nemu, "NEMU_HOME", self.nemu_home, self.set_nemu_home), 64 (args.am, "AM_HOME", self.am_home, self.set_am_home), 65 (args.dramsim3, "DRAMSIM3_HOME", self.dramsim3_home, self.set_dramsim3_home), 66 (args.rvtest, "RVTEST_HOME", self.rvtest_home, self.set_rvtest_home), 67 ] 68 for (arg_in, env, default, set_func) in all_path: 69 set_func(self.__extract_path(arg_in, env, default)) 70 # Chisel arguments 71 self.enable_log = args.enable_log 72 self.num_cores = args.num_cores 73 # Makefile arguments 74 self.threads = args.threads 75 self.with_dramsim3 = 1 if args.with_dramsim3 else None 76 self.is_release = 1 if args.release else None 77 self.is_spike = "Spike" if args.spike else None 78 self.trace = 1 if args.trace or not args.disable_fork and not args.trace_fst else None 79 self.trace_fst = "fst" if args.trace_fst else None 80 self.config = args.config 81 self.is_mfc = 1 if args.mfc else None 82 self.emu_optimize = args.emu_optimize 83 # emu arguments 84 self.max_instr = args.max_instr 85 self.ram_size = args.ram_size 86 self.seed = random.randint(0, 9999) 87 self.numa = args.numa 88 self.diff = args.diff 89 if args.spike and "nemu" in args.diff: 90 self.diff = self.diff.replace("nemu-interpreter", "spike") 91 self.fork = not args.disable_fork 92 self.disable_diff = args.no_diff 93 self.disable_db = args.no_db 94 # wave dump path 95 if args.wave_dump is not None: 96 self.set_wave_home(args.wave_dump) 97 else: 98 self.set_wave_home(self.default_wave_home) 99 100 def get_env_variables(self): 101 all_env = { 102 "NOOP_HOME" : self.noop_home, 103 "NEMU_HOME" : self.nemu_home, 104 "WAVE_HOME" : self.wave_home, 105 "AM_HOME" : self.am_home, 106 "DRAMSIM3_HOME": self.dramsim3_home, 107 "MODULEPATH": "/usr/share/Modules/modulefiles:/etc/modulefiles" 108 } 109 return all_env 110 111 def get_chisel_args(self, prefix=None): 112 chisel_args = [ 113 (self.enable_log, "enable-log") 114 ] 115 args = map(lambda x: x[1], filter(lambda arg: arg[0], chisel_args)) 116 if prefix is not None: 117 args = map(lambda x: prefix + x, args) 118 return args 119 120 def get_makefile_args(self): 121 makefile_args = [ 122 (self.threads, "EMU_THREADS"), 123 (self.with_dramsim3, "WITH_DRAMSIM3"), 124 (self.is_release, "RELEASE"), 125 (self.is_spike, "REF"), 126 (self.trace, "EMU_TRACE"), 127 (self.trace_fst, "EMU_TRACE"), 128 (self.config, "CONFIG"), 129 (self.num_cores, "NUM_CORES"), 130 (self.is_mfc, "MFC"), 131 (self.emu_optimize, "EMU_OPTIMIZE") 132 ] 133 args = filter(lambda arg: arg[0] is not None, makefile_args) 134 return args 135 136 def get_emu_args(self): 137 emu_args = [ 138 (self.max_instr, "max-instr"), 139 (self.diff, "diff"), 140 (self.seed, "seed"), 141 (self.ram_size, "ram-size"), 142 ] 143 args = filter(lambda arg: arg[0] is not None, emu_args) 144 return args 145 146 def show(self): 147 print("Extra environment variables:") 148 env = self.get_env_variables() 149 for env_name in env: 150 print(f"{env_name}: {env[env_name]}") 151 print() 152 print("Chisel arguments:") 153 print(" ".join(self.get_chisel_args())) 154 print() 155 print("Makefile arguments:") 156 for val, name in self.get_makefile_args(): 157 print(f"{name}={val}") 158 print() 159 print("emu arguments:") 160 for val, name in self.get_emu_args(): 161 print(f"--{name} {val}") 162 print() 163 164 def __extract_path(self, path, env=None, default=None): 165 if path is None and env is not None: 166 path = os.getenv(env) 167 if path is None and default is not None: 168 path = default 169 path = os.path.realpath(path) 170 return path 171 172 def set_noop_home(self, path): 173 self.noop_home = path 174 175 def set_nemu_home(self, path): 176 self.nemu_home = path 177 178 def set_am_home(self, path): 179 self.am_home = path 180 181 def set_dramsim3_home(self, path): 182 self.dramsim3_home = path 183 184 def set_rvtest_home(self, path): 185 self.rvtest_home = path 186 187 def set_wave_home(self, path): 188 print(f"set wave home to {path}") 189 self.wave_home = path 190 191# XiangShan environment 192class XiangShan(object): 193 def __init__(self, args): 194 self.args = XSArgs(args) 195 self.timeout = args.timeout 196 197 def show(self): 198 self.args.show() 199 200 def make_clean(self): 201 print("Clean up CI workspace") 202 self.show() 203 return_code = self.__exec_cmd(f'make -C $NOOP_HOME clean') 204 return return_code 205 206 def generate_verilog(self): 207 print("Generating XiangShan verilog with the following configurations:") 208 self.show() 209 sim_args = " ".join(self.args.get_chisel_args(prefix="--")) 210 make_args = " ".join(map(lambda arg: f"{arg[1]}={arg[0]}", self.args.get_makefile_args())) 211 return_code = self.__exec_cmd(f'make -C $NOOP_HOME verilog SIM_ARGS="{sim_args}" {make_args}') 212 return return_code 213 214 def generate_sim_verilog(self): 215 print("Generating XiangShan sim-verilog with the following configurations:") 216 self.show() 217 sim_args = " ".join(self.args.get_chisel_args(prefix="--")) 218 make_args = " ".join(map(lambda arg: f"{arg[1]}={arg[0]}", self.args.get_makefile_args())) 219 return_code = self.__exec_cmd(f'make -C $NOOP_HOME sim-verilog SIM_ARGS="{sim_args}" {make_args}') 220 return return_code 221 222 def build_emu(self): 223 print("Building XiangShan emu with the following configurations:") 224 self.show() 225 sim_args = " ".join(self.args.get_chisel_args(prefix="--")) 226 make_args = " ".join(map(lambda arg: f"{arg[1]}={arg[0]}", self.args.get_makefile_args())) 227 return_code = self.__exec_cmd(f'make -C $NOOP_HOME emu -j200 SIM_ARGS="{sim_args}" {make_args}') 228 return return_code 229 230 def build_simv(self): 231 print("Building XiangShan simv with the following configurations") 232 self.show() 233 make_args = " ".join(map(lambda arg: f"{arg[1]}={arg[0]}", self.args.get_makefile_args())) 234 # TODO: make the following commands grouped as unseen scripts 235 return_code = self.__exec_cmd(f'\ 236 eval `/usr/bin/modulecmd zsh load license`;\ 237 eval `/usr/bin/modulecmd zsh load synopsys/vcs/Q-2020.03-SP2`;\ 238 eval `/usr/bin/modulecmd zsh load synopsys/verdi/S-2021.09-SP1`;\ 239 VERDI_HOME=/nfs/tools/synopsys/verdi/S-2021.09-SP1 \ 240 make -C $NOOP_HOME simv {make_args} CONSIDER_FSDB=1') # set CONSIDER_FSDB for compatibility 241 return return_code 242 243 def run_emu(self, workload): 244 print("Running XiangShan emu with the following configurations:") 245 self.show() 246 emu_args = " ".join(map(lambda arg: f"--{arg[1]} {arg[0]}", self.args.get_emu_args())) 247 print("workload:", workload) 248 numa_args = "" 249 if self.args.numa: 250 numa_info = get_free_cores(self.args.threads) 251 numa_args = f"numactl -m {numa_info[0]} -C {numa_info[1]}-{numa_info[2]}" 252 fork_args = "--enable-fork" if self.args.fork else "" 253 diff_args = "--no-diff" if self.args.disable_diff else "" 254 chiseldb_args = "--dump-db" if not self.args.disable_db else "" 255 return_code = self.__exec_cmd(f'ulimit -s {32 * 1024}; {numa_args} $NOOP_HOME/build/emu -i {workload} {emu_args} {fork_args} {diff_args} {chiseldb_args}') 256 return return_code 257 258 def run_simv(self, workload): 259 print("Running XiangShan simv with the following configurations:") 260 self.show() 261 diff_args = "$NOOP_HOME/"+ args.diff 262 return_code = self.__exec_cmd(f'$NOOP_HOME/difftest/simv +workload={workload} +diff={diff_args}') 263 return return_code 264 265 def run(self, args): 266 if args.ci is not None: 267 return self.run_ci(args.ci) 268 if args.ci_vcs is not None: 269 return self.run_ci_vcs(args.ci_vcs) 270 actions = [ 271 (args.generate, lambda _ : self.generate_verilog()), 272 (args.vcs_gen, lambda _ : self.generate_sim_verilog()), 273 (args.build, lambda _ : self.build_emu()), 274 (args.vcs_build, lambda _ : self.build_simv()), 275 (args.workload, lambda args: self.run_emu(args.workload)), 276 (args.clean, lambda _ : self.make_clean()) 277 ] 278 valid_actions = map(lambda act: act[1], filter(lambda act: act[0], actions)) 279 for i, action in enumerate(valid_actions): 280 print(f"Action {i}:") 281 ret = action(args) 282 if ret: 283 return ret 284 return 0 285 286 def __exec_cmd(self, cmd): 287 env = dict(os.environ) 288 env.update(self.args.get_env_variables()) 289 print("subprocess call cmd:", cmd) 290 start = time.time() 291 proc = subprocess.Popen(cmd, shell=True, env=env, preexec_fn=os.setsid) 292 try: 293 return_code = proc.wait(self.timeout) 294 end = time.time() 295 print(f"Elapsed time: {end - start} seconds") 296 return return_code 297 except (KeyboardInterrupt, subprocess.TimeoutExpired): 298 os.killpg(os.getpgid(proc.pid), signal.SIGINT) 299 print(f"KeyboardInterrupt or TimeoutExpired.") 300 return 0 301 302 def __get_ci_cputest(self, name=None): 303 base_dir = os.path.join(self.args.am_home, "tests/cputest/build") 304 cputest = os.listdir(base_dir) 305 cputest = filter(lambda x: x.endswith(".bin"), cputest) 306 cputest = map(lambda x: os.path.join(base_dir, x), cputest) 307 return cputest 308 309 def __get_ci_rvtest(self, name=None): 310 base_dir = os.path.join(self.args.rvtest_home, "isa/build") 311 riscv_tests = os.listdir(base_dir) 312 riscv_tests = filter(lambda x: x.endswith(".bin"), riscv_tests) 313 all_rv_tests = ["rv64ui", "rv64um", "rv64ua", "rv64uf", "rv64ud"] 314 riscv_tests = filter(lambda x: x[:6] in all_rv_tests, riscv_tests) 315 riscv_tests = map(lambda x: os.path.join(base_dir, x), riscv_tests) 316 return riscv_tests 317 318 def __get_ci_misc(self, name=None): 319 base_dir = "/nfs/home/share/ci-workloads" 320 workloads = [ 321 "bitmanip/bitMisc.bin", 322 "crypto/crypto-riscv64-noop.bin", 323 "coremark_rv64gc_o2/coremark-riscv64-xs.bin", 324 "coremark_rv64gc_o3/coremark-riscv64-xs.bin", 325 "coremark_rv64gcb_o3/coremark-riscv64-xs.bin", 326 "ext_intr/amtest-riscv64-xs.bin", 327 "cache-alias/aliastest-riscv64-xs.bin", 328 "Svinval/rv64mi-p-svinval.bin", 329 "pmp/pmp.riscv.bin", 330 "pmp/pmp-am/amtest-riscv64-xs.bin", 331 "pmp/hugepage-pmp-atom/amtest-riscv64-xs.bin", 332 "asid/asid.bin", 333 "isa_misc/xret_clear_mprv.bin", 334 "isa_misc/satp_ppn.bin", 335 "cache-management/softprefetchtest-riscv64-xs.bin" 336 ] 337 misc_tests = map(lambda x: os.path.join(base_dir, x), workloads) 338 return misc_tests 339 340 def __get_ci_mc(self, name=None): 341 base_dir = "/nfs/home/share/ci-workloads" 342 workloads = [ 343 "dualcoretest/ldvio-riscv64-xs.bin" 344 ] 345 mc_tests = map(lambda x: os.path.join(base_dir, x), workloads) 346 return mc_tests 347 348 def __get_ci_nodiff(self, name=None): 349 base_dir = "/nfs/home/share/ci-workloads" 350 workloads = [ 351 "cache-management/cacheoptest-riscv64-xs.bin" 352 ] 353 tests = map(lambda x: os.path.join(base_dir, x), workloads) 354 return tests 355 356 def __am_apps_path(self, bench): 357 filename = f"{bench}-riscv64-noop.bin" 358 return [os.path.join(self.args.am_home, "apps", bench, "build", filename)] 359 360 def __get_ci_workloads(self, name): 361 workloads = { 362 "linux-hello": "bbl.bin", 363 "linux-hello-smp": "bbl.bin", 364 "linux-hello-opensbi": "fw_payload.bin", 365 "linux-hello-smp-opensbi": "fw_payload.bin", 366 "povray": "_700480000000_.gz", 367 "mcf": "_17520000000_.gz", 368 "xalancbmk": "_266100000000_.gz", 369 "gcc": "_39720000000_.gz", 370 "namd": "_434640000000_.gz", 371 "milc": "_103620000000_.gz", 372 "lbm": "_140840000000_.gz", 373 "gromacs": "_275480000000_.gz", 374 "wrf": "_1916220000000_.gz", 375 "astar": "_122060000000_.gz" 376 } 377 if name in workloads: 378 return [os.path.join("/nfs/home/share/ci-workloads", name, workloads[name])] 379 # select a random SPEC checkpoint 380 assert(name == "random") 381 all_cpt = [ 382 "/nfs-nvme/home/share/checkpoints_profiles/spec06_rv64gcb_o2_20m/take_cpt", 383 "/nfs-nvme/home/share/checkpoints_profiles/spec06_rv64gcb_o3_20m/take_cpt", 384 "/nfs-nvme/home/share/checkpoints_profiles/spec06_rv64gc_o2_20m/take_cpt", 385 "/nfs-nvme/home/share/checkpoints_profiles/spec06_rv64gc_o2_50m/take_cpt", 386 "/nfs-nvme/home/share/checkpoints_profiles/spec17_rv64gcb_o2_20m/take_cpt", 387 "/nfs-nvme/home/share/checkpoints_profiles/spec17_rv64gcb_o3_20m/take_cpt", 388 "/nfs-nvme/home/share/checkpoints_profiles/spec17_rv64gc_o2_50m/take_cpt", 389 "/nfs-nvme/home/share/checkpoints_profiles/spec17_speed_rv64gcb_o3_20m/take_cpt" 390 ] 391 all_json = [ 392 "/nfs-nvme/home/share/checkpoints_profiles/spec06_rv64gcb_o2_20m/json/simpoint_summary.json", 393 "/nfs-nvme/home/share/checkpoints_profiles/spec06_rv64gcb_o3_20m/simpoint_summary.json", 394 "/nfs-nvme/home/share/checkpoints_profiles/spec06_rv64gc_o2_20m/simpoint_summary.json", 395 "/nfs-nvme/home/share/checkpoints_profiles/spec06_rv64gc_o2_50m/simpoint_summary.json", 396 "/nfs-nvme/home/share/checkpoints_profiles/spec17_rv64gcb_o2_20m/simpoint_summary.json", 397 "/nfs-nvme/home/share/checkpoints_profiles/spec17_rv64gcb_o3_20m/simpoint_summary.json", 398 "/nfs-nvme/home/share/checkpoints_profiles/spec17_rv64gc_o2_50m/simpoint_summary.json", 399 "/nfs-nvme/home/share/checkpoints_profiles/spec17_speed_rv64gcb_o3_20m/simpoint_summary.json" 400 ] 401 assert(len(all_cpt) == len(all_json)) 402 cpt_path, json_path = random.choice(list(zip(all_cpt, all_json))) 403 all_gcpt = load_all_gcpt(cpt_path, json_path) 404 return [random.choice(all_gcpt)] 405 406 def run_ci(self, test): 407 all_tests = { 408 "cputest": self.__get_ci_cputest, 409 "riscv-tests": self.__get_ci_rvtest, 410 "misc-tests": self.__get_ci_misc, 411 "mc-tests": self.__get_ci_mc, 412 "nodiff-tests": self.__get_ci_nodiff, 413 "microbench": self.__am_apps_path, 414 "coremark": self.__am_apps_path 415 } 416 for target in all_tests.get(test, self.__get_ci_workloads)(test): 417 print(target) 418 ret = self.run_emu(target) 419 if ret: 420 if self.args.default_wave_home != self.args.wave_home: 421 print("copy wave file to " + self.args.wave_home) 422 self.__exec_cmd(f"cp $NOOP_HOME/build/*.vcd $WAVE_HOME") 423 self.__exec_cmd(f"cp $NOOP_HOME/build/emu $WAVE_HOME") 424 self.__exec_cmd(f"cp $NOOP_HOME/build/rtl/SimTop.v $WAVE_HOME") 425 self.__exec_cmd(f"cp $NOOP_HOME/build/*.db $WAVE_HOME") 426 return ret 427 return 0 428 429 def run_ci_vcs(self, test): 430 all_tests = { 431 "cputest": self.__get_ci_cputest, 432 "riscv-tests": self.__get_ci_rvtest, 433 "misc-tests": self.__get_ci_misc, 434 "mc-tests": self.__get_ci_mc, 435 "nodiff-tests": self.__get_ci_nodiff, 436 "microbench": self.__am_apps_path, 437 "coremark": self.__am_apps_path 438 } 439 for target in all_tests.get(test, self.__get_ci_workloads)(test): 440 print(target) 441 ret = self.run_simv(target) 442 if ret: 443 if self.args.default_wave_home != self.args.wave_home: 444 print("copy wave file to " + self.args.wave_home) 445 self.__exec_cmd(f"cp $NOOP_HOME/build/*.vcd $WAVE_HOME") 446 self.__exec_cmd(f"cp $NOOP_HOME/build/emu $WAVE_HOME") 447 self.__exec_cmd(f"cp $NOOP_HOME/build/rtl/SimTop.v $WAVE_HOME") 448 self.__exec_cmd(f"cp $NOOP_HOME/build/*.db $WAVE_HOME") 449 return ret 450 return 0 451 452def get_free_cores(n): 453 while True: 454 # To avoid potential conflicts, we allow CI to use SMT. 455 num_logical_core = psutil.cpu_count(logical=False) 456 core_usage = psutil.cpu_percent(interval=1, percpu=True) 457 num_window = num_logical_core // n 458 for i in range(num_window): 459 window_usage = core_usage[i * n : i * n + n] 460 if sum(window_usage) < 30 * n and True not in map(lambda x: x > 90, window_usage): 461 return (((i * n) % 128)// 64, i * n, i * n + n - 1) 462 print(f"No free {n} cores found. CPU usage: {core_usage}\n") 463 464if __name__ == "__main__": 465 parser = argparse.ArgumentParser(description='Python wrapper for XiangShan') 466 parser.add_argument('workload', nargs='?', type=str, default="", 467 help='input workload file in binary format') 468 # actions 469 parser.add_argument('--build', action='store_true', help='build XS emu') 470 parser.add_argument('--generate', action='store_true', help='generate XS verilog') 471 parser.add_argument('--vcs-gen', action='store_true', help='generate XS sim verilog for vcs') 472 parser.add_argument('--vcs-build', action='store_true', help='build XS simv') 473 parser.add_argument('--ci', nargs='?', type=str, const="", help='run CI tests') 474 parser.add_argument('--ci-vcs', nargs='?', type=str, const="", help='run CI tests on simv') 475 parser.add_argument('--clean', action='store_true', help='clean up XiangShan CI workspace') 476 parser.add_argument('--timeout', nargs='?', type=int, default=None, help='timeout (in seconds)') 477 # environment variables 478 parser.add_argument('--nemu', nargs='?', type=str, help='path to nemu') 479 parser.add_argument('--am', nargs='?', type=str, help='path to nexus-am') 480 parser.add_argument('--dramsim3', nargs='?', type=str, help='path to dramsim3') 481 parser.add_argument('--rvtest', nargs='?', type=str, help='path to riscv-tests') 482 parser.add_argument('--wave-dump', nargs='?', type=str , help='path to dump wave') 483 # chisel arguments 484 parser.add_argument('--enable-log', action='store_true', help='enable log') 485 parser.add_argument('--num-cores', type=int, help='number of cores') 486 # makefile arguments 487 parser.add_argument('--release', action='store_true', help='enable release') 488 parser.add_argument('--spike', action='store_true', help='enable spike diff') 489 parser.add_argument('--with-dramsim3', action='store_true', help='enable dramsim3') 490 parser.add_argument('--threads', nargs='?', type=int, help='number of emu threads') 491 parser.add_argument('--trace', action='store_true', help='enable vcd waveform') 492 parser.add_argument('--trace-fst', action='store_true', help='enable fst waveform') 493 parser.add_argument('--config', nargs='?', type=str, help='config') 494 parser.add_argument('--mfc', action='store_true', help='use mfc') 495 parser.add_argument('--emu-optimize', nargs='?', type=str, help='verilator optimization letter') 496 # emu arguments 497 parser.add_argument('--numa', action='store_true', help='use numactl') 498 parser.add_argument('--diff', nargs='?', default="./ready-to-run/riscv64-nemu-interpreter-so", type=str, help='nemu so') 499 parser.add_argument('--max-instr', nargs='?', type=int, help='max instr') 500 parser.add_argument('--disable-fork', action='store_true', help='disable lightSSS') 501 parser.add_argument('--no-diff', action='store_true', help='disable difftest') 502 parser.add_argument('--ram-size', nargs='?', type=str, help='manually set simulation memory size (8GB by default)') 503 parser.add_argument('--no-db', action='store_true', help='disable chiseldb dump') 504 505 args = parser.parse_args() 506 507 xs = XiangShan(args) 508 ret = xs.run(args) 509 510 sys.exit(ret) 511