1#*************************************************************************************** 2# Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3# Copyright (c) 2020-2021 Peng Cheng Laboratory 4# 5# XiangShan is licensed under Mulan PSL v2. 6# You can use this software according to the terms and conditions of the Mulan PSL v2. 7# You may obtain a copy of Mulan PSL v2 at: 8# http://license.coscl.org.cn/MulanPSL2 9# 10# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13# 14# See the Mulan PSL v2 for more details. 15#*************************************************************************************** 16 17# Simple version of xiangshan python wrapper 18 19import argparse 20import os 21import random 22import subprocess 23import sys 24import time 25 26import psutil 27 28 29class XSArgs(object): 30 script_path = os.path.realpath(__file__) 31 # default path to the repositories 32 noop_home = os.path.join(os.path.dirname(script_path), "..") 33 nemu_home = os.path.join(noop_home, "../NEMU") 34 am_home = os.path.join(noop_home, "../nexus-am") 35 dramsim3_home = os.path.join(noop_home, "../DRAMsim3") 36 rvtest_home = os.path.join(noop_home, "../riscv-tests") 37 default_wave_home = os.path.join(noop_home, "build") 38 wave_home = default_wave_home 39 40 def __init__(self, args): 41 # all path environment variables that should be set 42 all_path = [ 43 # (python argument, environment variable, default, target function) 44 (None, "NOOP_HOME", self.noop_home, self.set_noop_home), 45 (args.nemu, "NEMU_HOME", self.nemu_home, self.set_nemu_home), 46 (args.am, "AM_HOME", self.am_home, self.set_am_home), 47 (args.dramsim3, "DRAMSIM3_HOME", self.dramsim3_home, self.set_dramsim3_home), 48 (args.rvtest, "RVTEST_HOME", self.rvtest_home, self.set_rvtest_home), 49 ] 50 for (arg_in, env, default, set_func) in all_path: 51 set_func(self.__extract_path(arg_in, env, default)) 52 # Chisel arguments 53 self.enable_log = args.enable_log 54 self.num_cores = args.num_cores 55 # Makefile arguments 56 self.threads = args.threads 57 self.with_dramsim3 = 1 if args.with_dramsim3 else None 58 self.is_release = 1 if args.release else None 59 self.trace = 1 if args.trace or not args.disable_fork else None 60 self.config = args.config 61 # emu arguments 62 self.max_instr = args.max_instr 63 self.seed = random.randint(0, 9999) 64 self.numa = args.numa 65 self.diff = args.diff 66 self.fork = not args.disable_fork 67 self.disable_diff = args.no_diff 68 # wave dump path 69 if args.wave_dump is not None: 70 self.set_wave_home(args.wave_dump) 71 else: 72 self.set_wave_home(self.default_wave_home) 73 74 def get_env_variables(self): 75 all_env = { 76 "NOOP_HOME" : self.noop_home, 77 "NEMU_HOME" : self.nemu_home, 78 "WAVE_HOME" : self.wave_home, 79 "AM_HOME" : self.am_home, 80 "DRAMSIM3_HOME": self.dramsim3_home 81 } 82 return all_env 83 84 def get_chisel_args(self, prefix=None): 85 chisel_args = [ 86 (self.enable_log, "enable-log") 87 ] 88 args = map(lambda x: x[1], filter(lambda arg: arg[0], chisel_args)) 89 if prefix is not None: 90 args = map(lambda x: prefix + x, args) 91 return args 92 93 def get_makefile_args(self): 94 makefile_args = [ 95 (self.threads, "EMU_THREADS"), 96 (self.with_dramsim3, "WITH_DRAMSIM3"), 97 (self.is_release, "RELEASE"), 98 (self.trace, "EMU_TRACE"), 99 (self.config, "CONFIG"), 100 (self.num_cores, "NUM_CORES") 101 ] 102 args = filter(lambda arg: arg[0] is not None, makefile_args) 103 return args 104 105 def get_emu_args(self): 106 emu_args = [ 107 (self.max_instr, "max-instr"), 108 (self.diff, "diff"), 109 (self.seed, "seed") 110 ] 111 args = filter(lambda arg: arg[0] is not None, emu_args) 112 return args 113 114 def show(self): 115 print("Extra environment variables:") 116 env = self.get_env_variables() 117 for env_name in env: 118 print(f"{env_name}: {env[env_name]}") 119 print() 120 print("Chisel arguments:") 121 print(" ".join(self.get_chisel_args())) 122 print() 123 print("Makefile arguments:") 124 for val, name in self.get_makefile_args(): 125 print(f"{name}={val}") 126 print() 127 print("emu arguments:") 128 for val, name in self.get_emu_args(): 129 print(f"--{name} {val}") 130 print() 131 132 def __extract_path(self, path, env=None, default=None): 133 if path is None and env is not None: 134 path = os.getenv(env) 135 if path is None and default is not None: 136 path = default 137 path = os.path.realpath(path) 138 return path 139 140 def set_noop_home(self, path): 141 self.noop_home = path 142 143 def set_nemu_home(self, path): 144 self.nemu_home = path 145 146 def set_am_home(self, path): 147 self.am_home = path 148 149 def set_dramsim3_home(self, path): 150 self.dramsim3_home = path 151 152 def set_rvtest_home(self, path): 153 self.rvtest_home = path 154 155 def set_wave_home(self, path): 156 print(f"set wave home to {path}") 157 self.wave_home = path 158 159# XiangShan environment 160class XiangShan(object): 161 def __init__(self, args): 162 self.args = XSArgs(args) 163 164 def show(self): 165 self.args.show() 166 167 def make_clean(self): 168 print("Clean up CI workspace") 169 self.show() 170 return_code = self.__exec_cmd(f'make -C $NOOP_HOME clean') 171 return return_code 172 173 def generate_verilog(self): 174 print("Generating XiangShan verilog with the following configurations:") 175 self.show() 176 sim_args = " ".join(self.args.get_chisel_args(prefix="--")) 177 make_args = " ".join(map(lambda arg: f"{arg[1]}={arg[0]}", self.args.get_makefile_args())) 178 return_code = self.__exec_cmd(f'make -C $NOOP_HOME verilog SIM_ARGS="{sim_args}" {make_args}') 179 return return_code 180 181 def build_emu(self): 182 print("Building XiangShan emu with the following configurations:") 183 self.show() 184 sim_args = " ".join(self.args.get_chisel_args(prefix="--")) 185 make_args = " ".join(map(lambda arg: f"{arg[1]}={arg[0]}", self.args.get_makefile_args())) 186 return_code = self.__exec_cmd(f'make -C $NOOP_HOME emu -j200 SIM_ARGS="{sim_args}" {make_args}') 187 return return_code 188 189 def run_emu(self, workload): 190 print("Running XiangShan emu with the following configurations:") 191 self.show() 192 emu_args = " ".join(map(lambda arg: f"--{arg[1]} {arg[0]}", self.args.get_emu_args())) 193 print("workload:", workload) 194 numa_args = "" 195 if self.args.numa: 196 numa_info = get_free_cores(self.args.threads) 197 numa_args = f"numactl -m {numa_info[0]} -C {numa_info[1]}-{numa_info[2]}" 198 fork_args = "--enable-fork" if self.args.fork else "" 199 diff_args = "--no-diff" if self.args.disable_diff else "" 200 return_code = self.__exec_cmd(f'{numa_args} $NOOP_HOME/build/emu -i {workload} {emu_args} {fork_args} {diff_args}') 201 return return_code 202 203 def run(self, args): 204 if args.ci is not None: 205 return self.run_ci(args.ci) 206 actions = [ 207 (args.generate, lambda _ : self.generate_verilog()), 208 (args.build, lambda _ : self.build_emu()), 209 (args.workload, lambda args: self.run_emu(args.workload)), 210 (args.clean, lambda _ : self.make_clean()) 211 ] 212 valid_actions = map(lambda act: act[1], filter(lambda act: act[0], actions)) 213 for i, action in enumerate(valid_actions): 214 print(f"Action {i}:") 215 ret = action(args) 216 if ret: 217 return ret 218 return 0 219 220 def __exec_cmd(self, cmd): 221 env = dict(os.environ) 222 env.update(self.args.get_env_variables()) 223 print("subprocess call cmd:", cmd) 224 start = time.time() 225 return_code = subprocess.call(cmd, shell=True, env=env) 226 end = time.time() 227 print(f"Elapsed time: {end - start} seconds") 228 return return_code 229 230 def __get_ci_cputest(self, name=None): 231 base_dir = os.path.join(self.args.am_home, "tests/cputest/build") 232 cputest = os.listdir(base_dir) 233 cputest = filter(lambda x: x.endswith(".bin"), cputest) 234 cputest = map(lambda x: os.path.join(base_dir, x), cputest) 235 return cputest 236 237 def __get_ci_rvtest(self, name=None): 238 base_dir = os.path.join(self.args.rvtest_home, "isa/build") 239 riscv_tests = os.listdir(base_dir) 240 riscv_tests = filter(lambda x: x.endswith(".bin"), riscv_tests) 241 all_rv_tests = ["rv64ui", "rv64um", "rv64ua", "rv64uf", "rv64ud"] 242 riscv_tests = filter(lambda x: x[:6] in all_rv_tests, riscv_tests) 243 riscv_tests = map(lambda x: os.path.join(base_dir, x), riscv_tests) 244 return riscv_tests 245 246 def __get_ci_misc(self, name=None): 247 base_dir = "/nfs/home/share/ci-workloads" 248 workloads = [ 249 "bitmanip/bitMisc.bin", 250 "crypto/crypto-riscv64-noop.bin", 251 "coremark_rv64gc_o2/coremark-riscv64-xs.bin", 252 "coremark_rv64gc_o3/coremark-riscv64-xs.bin", 253 "coremark_rv64gcb_o3/coremark-riscv64-xs.bin", 254 "ext_intr/amtest-riscv64-xs.bin", 255 "cache-alias/aliastest-riscv64-xs.bin", 256 "Svinval/rv64mi-p-svinval.bin", 257 "pmp/pmp.riscv.bin", 258 "asid/asid.bin", 259 "isa_misc/xret_clear_mprv.bin", 260 "isa_misc/satp_ppn.bin", 261 "cache-management/softprefetchtest-riscv64-xs.bin" 262 ] 263 misc_tests = map(lambda x: os.path.join(base_dir, x), workloads) 264 return misc_tests 265 266 def __get_ci_mc(self, name=None): 267 base_dir = "/nfs/home/share/ci-workloads" 268 workloads = [ 269 "dualcoretest/ldvio-riscv64-xs.bin" 270 ] 271 mc_tests = map(lambda x: os.path.join(base_dir, x), workloads) 272 return mc_tests 273 274 def __get_ci_nodiff(self, name=None): 275 base_dir = "/nfs/home/share/ci-workloads" 276 workloads = [ 277 "cache-management/cacheoptest-riscv64-xs.bin" 278 ] 279 tests = map(lambda x: os.path.join(base_dir, x), workloads) 280 return tests 281 282 def __am_apps_path(self, bench): 283 filename = f"{bench}-riscv64-noop.bin" 284 return [os.path.join(self.args.am_home, "apps", bench, "build", filename)] 285 286 def __get_ci_workloads(self, name): 287 workloads = { 288 "linux-hello": "bbl.bin", 289 "linux-hello-smp": "bbl.bin", 290 "povray": "_700480000000_.gz", 291 "mcf": "_17520000000_.gz", 292 "xalancbmk": "_266100000000_.gz", 293 "gcc": "_39720000000_.gz", 294 "namd": "_434640000000_.gz", 295 "milc": "_103620000000_.gz", 296 "lbm": "_140840000000_.gz", 297 "gromacs": "_275480000000_.gz", 298 "wrf": "_1916220000000_.gz", 299 "astar": "_122060000000_.gz" 300 } 301 return [os.path.join("/nfs/home/share/ci-workloads", name, workloads[name])] 302 303 def run_ci(self, test): 304 all_tests = { 305 "cputest": self.__get_ci_cputest, 306 "riscv-tests": self.__get_ci_rvtest, 307 "misc-tests": self.__get_ci_misc, 308 "mc-tests": self.__get_ci_mc, 309 "nodiff-tests": self.__get_ci_nodiff, 310 "microbench": self.__am_apps_path, 311 "coremark": self.__am_apps_path 312 } 313 for target in all_tests.get(test, self.__get_ci_workloads)(test): 314 print(target) 315 ret = self.run_emu(target) 316 if ret: 317 if self.args.default_wave_home != self.args.wave_home: 318 print("copy wave file to " + self.args.wave_home) 319 self.__exec_cmd(f"cp $NOOP_HOME/build/*.vcd $WAVE_HOME") 320 self.__exec_cmd(f"cp $NOOP_HOME/build/emu $WAVE_HOME") 321 self.__exec_cmd(f"cp $NOOP_HOME/build/SimTop.v $WAVE_HOME") 322 return ret 323 return 0 324 325def get_free_cores(n): 326 while True: 327 # To avoid potential conflicts, we allow CI to use SMT. 328 num_logical_core = psutil.cpu_count(logical=True) 329 core_usage = psutil.cpu_percent(interval=1, percpu=True) 330 num_window = num_logical_core // n 331 for i in range(num_window): 332 window_usage = core_usage[i * n : i * n + n] 333 if sum(window_usage) < 0.3 * n and True not in map(lambda x: x > 0.5, window_usage): 334 return (((i * n) % 128)// 64, i * n, i * n + n - 1) 335 print(f"No free {n} cores found. CPU usage: {core_usage}\n") 336 337if __name__ == "__main__": 338 parser = argparse.ArgumentParser(description='Python wrapper for XiangShan') 339 parser.add_argument('workload', nargs='?', type=str, default="", 340 help='input workload file in binary format') 341 # actions 342 parser.add_argument('--build', action='store_true', help='build XS emu') 343 parser.add_argument('--generate', action='store_true', help='generate XS verilog') 344 parser.add_argument('--ci', nargs='?', type=str, const="", help='run CI tests') 345 parser.add_argument('--clean', action='store_true', help='clean up XiangShan CI workspace') 346 # environment variables 347 parser.add_argument('--nemu', nargs='?', type=str, help='path to nemu') 348 parser.add_argument('--am', nargs='?', type=str, help='path to nexus-am') 349 parser.add_argument('--dramsim3', nargs='?', type=str, help='path to dramsim3') 350 parser.add_argument('--rvtest', nargs='?', type=str, help='path to riscv-tests') 351 parser.add_argument('--wave-dump', nargs='?', type=str , help='path to dump wave') 352 # chisel arguments 353 parser.add_argument('--enable-log', action='store_true', help='enable log') 354 parser.add_argument('--num-cores', type=int, help='number of cores') 355 # makefile arguments 356 parser.add_argument('--release', action='store_true', help='enable release') 357 parser.add_argument('--with-dramsim3', action='store_true', help='enable dramsim3') 358 parser.add_argument('--threads', nargs='?', type=int, help='number of emu threads') 359 parser.add_argument('--trace', action='store_true', help='enable waveform') 360 parser.add_argument('--config', nargs='?', type=str, help='config') 361 # emu arguments 362 parser.add_argument('--numa', action='store_true', help='use numactl') 363 parser.add_argument('--diff', nargs='?', default="./ready-to-run/riscv64-nemu-interpreter-so", type=str, help='nemu so') 364 parser.add_argument('--max-instr', nargs='?', type=int, help='max instr') 365 parser.add_argument('--disable-fork', action='store_true', help='disable lightSSS') 366 parser.add_argument('--no-diff', action='store_true', help='disable difftest') 367 # ci action head sha 368 369 args = parser.parse_args() 370 371 xs = XiangShan(args) 372 ret = xs.run(args) 373 374 sys.exit(ret) 375