xref: /XiangShan/scripts/xiangshan.py (revision 039cdc35f5f3b68b6295ec5ace90f22a77322e02)
1#***************************************************************************************
2# Copyright (c) 2024 Beijing Institute of Open Source Chip (BOSC)
3# Copyright (c) 2020-2024 Institute of Computing Technology, Chinese Academy of Sciences
4# Copyright (c) 2020-2021 Peng Cheng Laboratory
5#
6# XiangShan is licensed under Mulan PSL v2.
7# You can use this software according to the terms and conditions of the Mulan PSL v2.
8# You may obtain a copy of Mulan PSL v2 at:
9#          http://license.coscl.org.cn/MulanPSL2
10#
11# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
12# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
13# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
14#
15# See the Mulan PSL v2 for more details.
16#***************************************************************************************
17
18# Simple version of xiangshan python wrapper
19
20import argparse
21import json
22import os
23import random
24import signal
25import subprocess
26import sys
27import time
28import shlex
29import psutil
30
31
32def load_all_gcpt(gcpt_path, json_path):
33    all_gcpt = []
34    with open(json_path) as f:
35        data = json.load(f)
36    for benchspec in data:
37        for point in data[benchspec]:
38            weight = data[benchspec][point]
39            gcpt = os.path.join(gcpt_path, "_".join([benchspec, point, weight]))
40            bin_dir = os.path.join(gcpt, "0")
41            bin_file = list(os.listdir(bin_dir))
42            assert(len(bin_file) == 1)
43            bin_path = os.path.join(bin_dir, bin_file[0])
44            assert(os.path.isfile(bin_path))
45            all_gcpt.append(bin_path)
46    return all_gcpt
47
48class XSArgs(object):
49    script_path = os.path.realpath(__file__)
50    # default path to the repositories
51    noop_home = os.path.join(os.path.dirname(script_path), "..")
52    nemu_home = os.path.join(noop_home, "../NEMU")
53    am_home = os.path.join(noop_home, "../nexus-am")
54    dramsim3_home = os.path.join(noop_home, "../DRAMsim3")
55    rvtest_home = os.path.join(noop_home, "../riscv-tests")
56    default_wave_home = os.path.join(noop_home, "build")
57    wave_home   = default_wave_home
58
59    def __init__(self, args):
60        # all path environment variables that should be set
61        all_path = [
62            # (python argument, environment variable, default, target function)
63            (None, "NOOP_HOME", self.noop_home, self.set_noop_home),
64            (args.nemu, "NEMU_HOME", self.nemu_home, self.set_nemu_home),
65            (args.am, "AM_HOME", self.am_home, self.set_am_home),
66            (args.dramsim3, "DRAMSIM3_HOME", self.dramsim3_home, self.set_dramsim3_home),
67            (args.rvtest, "RVTEST_HOME", self.rvtest_home, self.set_rvtest_home),
68        ]
69        for (arg_in, env, default, set_func) in all_path:
70            set_func(self.__extract_path(arg_in, env, default))
71        # Chisel arguments
72        self.enable_log = args.enable_log
73        self.num_cores = args.num_cores
74        # Makefile arguments
75        self.threads = args.threads
76        self.with_dramsim3 = 1 if args.with_dramsim3 else None
77        self.is_release = 1 if args.release else None
78        self.is_spike = "Spike" if args.spike else None
79        self.trace = 1 if args.trace or not args.disable_fork and not args.trace_fst else None
80        self.trace_fst = "fst" if args.trace_fst else None
81        self.config = args.config
82        self.is_mfc = 1 if args.mfc else None
83        self.emu_optimize = args.emu_optimize
84        self.xprop = 1 if args.xprop else None
85        self.with_chiseldb = 0 if args.no_db else None
86        # emu arguments
87        self.max_instr = args.max_instr
88        self.ram_size = args.ram_size
89        self.seed = random.randint(0, 9999)
90        self.numa = args.numa
91        self.diff = args.diff
92        if args.spike and "nemu" in args.diff:
93            self.diff = self.diff.replace("nemu-interpreter", "spike")
94        self.fork = not args.disable_fork
95        self.disable_diff = args.no_diff
96        self.disable_db = args.no_db
97        self.pgo = args.pgo
98        self.pgo_max_cycle = args.pgo_max_cycle
99        self.pgo_emu_args = args.pgo_emu_args
100        self.llvm_profdata = args.llvm_profdata
101        # wave dump path
102        if args.wave_dump is not None:
103            self.set_wave_home(args.wave_dump)
104        else:
105            self.set_wave_home(self.default_wave_home)
106
107    def get_env_variables(self):
108        all_env = {
109            "NOOP_HOME"    : self.noop_home,
110            "NEMU_HOME"    : self.nemu_home,
111            "WAVE_HOME"    : self.wave_home,
112            "AM_HOME"      : self.am_home,
113            "DRAMSIM3_HOME": self.dramsim3_home,
114            "MODULEPATH": "/usr/share/Modules/modulefiles:/etc/modulefiles"
115        }
116        return all_env
117
118    def get_chisel_args(self, prefix=None):
119        chisel_args = [
120            (self.enable_log, "enable-log")
121        ]
122        args = map(lambda x: x[1], filter(lambda arg: arg[0], chisel_args))
123        if prefix is not None:
124            args = map(lambda x: prefix + x, args)
125        return args
126
127    def get_makefile_args(self):
128        makefile_args = [
129            (self.threads,       "EMU_THREADS"),
130            (self.with_dramsim3, "WITH_DRAMSIM3"),
131            (self.is_release,    "RELEASE"),
132            (self.is_spike,      "REF"),
133            (self.trace,         "EMU_TRACE"),
134            (self.trace_fst,     "EMU_TRACE"),
135            (self.config,        "CONFIG"),
136            (self.num_cores,     "NUM_CORES"),
137            (self.is_mfc,        "MFC"),
138            (self.emu_optimize,  "EMU_OPTIMIZE"),
139            (self.xprop,         "ENABLE_XPROP"),
140            (self.with_chiseldb, "WITH_CHISELDB"),
141            (self.pgo,           "PGO_WORKLOAD"),
142            (self.pgo_max_cycle, "PGO_MAX_CYCLE"),
143            (self.pgo_emu_args,  "PGO_EMU_ARGS"),
144            (self.llvm_profdata, "LLVM_PROFDATA"),
145        ]
146        args = filter(lambda arg: arg[0] is not None, makefile_args)
147        args = [(shlex.quote(str(arg[0])), arg[1]) for arg in args] # shell escape
148        return args
149
150    def get_emu_args(self):
151        emu_args = [
152            (self.max_instr, "max-instr"),
153            (self.diff,      "diff"),
154            (self.seed,      "seed"),
155            (self.ram_size,  "ram-size"),
156        ]
157        args = filter(lambda arg: arg[0] is not None, emu_args)
158        return args
159
160    def show(self):
161        print("Extra environment variables:")
162        env = self.get_env_variables()
163        for env_name in env:
164            print(f"{env_name}: {env[env_name]}")
165        print()
166        print("Chisel arguments:")
167        print(" ".join(self.get_chisel_args()))
168        print()
169        print("Makefile arguments:")
170        for val, name in self.get_makefile_args():
171            print(f"{name}={val}")
172        print()
173        print("emu arguments:")
174        for val, name in self.get_emu_args():
175            print(f"--{name} {val}")
176        print()
177
178    def __extract_path(self, path, env=None, default=None):
179        if path is None and env is not None:
180            path = os.getenv(env)
181        if path is None and default is not None:
182            path = default
183        path = os.path.realpath(path)
184        return path
185
186    def set_noop_home(self, path):
187        self.noop_home = path
188
189    def set_nemu_home(self, path):
190        self.nemu_home = path
191
192    def set_am_home(self, path):
193        self.am_home = path
194
195    def set_dramsim3_home(self, path):
196        self.dramsim3_home = path
197
198    def set_rvtest_home(self, path):
199        self.rvtest_home = path
200
201    def set_wave_home(self, path):
202        print(f"set wave home to {path}")
203        self.wave_home = path
204
205# XiangShan environment
206class XiangShan(object):
207    def __init__(self, args):
208        self.args = XSArgs(args)
209        self.timeout = args.timeout
210
211    def show(self):
212        self.args.show()
213
214    def make_clean(self):
215        print("Clean up CI workspace")
216        self.show()
217        return_code = self.__exec_cmd(f'make -C $NOOP_HOME clean')
218        return return_code
219
220    def generate_verilog(self):
221        print("Generating XiangShan verilog with the following configurations:")
222        self.show()
223        sim_args = " ".join(self.args.get_chisel_args(prefix="--"))
224        make_args = " ".join(map(lambda arg: f"{arg[1]}={arg[0]}", self.args.get_makefile_args()))
225        return_code = self.__exec_cmd(f'make -C $NOOP_HOME verilog SIM_ARGS="{sim_args}" {make_args}')
226        return return_code
227
228    def generate_sim_verilog(self):
229        print("Generating XiangShan sim-verilog with the following configurations:")
230        self.show()
231        sim_args = " ".join(self.args.get_chisel_args(prefix="--"))
232        make_args = " ".join(map(lambda arg: f"{arg[1]}={arg[0]}", self.args.get_makefile_args()))
233        return_code = self.__exec_cmd(f'make -C $NOOP_HOME sim-verilog SIM_ARGS="{sim_args}" {make_args}')
234        return return_code
235
236    def build_emu(self):
237        print("Building XiangShan emu with the following configurations:")
238        self.show()
239        sim_args = " ".join(self.args.get_chisel_args(prefix="--"))
240        make_args = " ".join(map(lambda arg: f"{arg[1]}={arg[0]}", self.args.get_makefile_args()))
241        return_code = self.__exec_cmd(f'make -C $NOOP_HOME emu -j200 SIM_ARGS="{sim_args}" {make_args}')
242        return return_code
243
244    def build_simv(self):
245        print("Building XiangShan simv with the following configurations")
246        self.show()
247        make_args = " ".join(map(lambda arg: f"{arg[1]}={arg[0]}", self.args.get_makefile_args()))
248        # TODO: make the following commands grouped as unseen scripts
249        return_code = self.__exec_cmd(f'\
250            eval `/usr/bin/modulecmd zsh load license`;\
251            eval `/usr/bin/modulecmd zsh load synopsys/vcs/Q-2020.03-SP2`;\
252            eval `/usr/bin/modulecmd zsh load synopsys/verdi/S-2021.09-SP1`;\
253            VERDI_HOME=/nfs/tools/synopsys/verdi/S-2021.09-SP1 \
254            make -C $NOOP_HOME simv {make_args} CONSIDER_FSDB=1')  # set CONSIDER_FSDB for compatibility
255        return return_code
256
257    def run_emu(self, workload):
258        print("Running XiangShan emu with the following configurations:")
259        self.show()
260        emu_args = " ".join(map(lambda arg: f"--{arg[1]} {arg[0]}", self.args.get_emu_args()))
261        print("workload:", workload)
262        numa_args = ""
263        if self.args.numa:
264            numa_info = get_free_cores(self.args.threads)
265            numa_args = f"numactl -m {numa_info[0]} -C {numa_info[1]}-{numa_info[2]}"
266        fork_args = "--enable-fork" if self.args.fork else ""
267        diff_args = "--no-diff" if self.args.disable_diff else ""
268        chiseldb_args = "--dump-db" if not self.args.disable_db else ""
269        return_code = self.__exec_cmd(f'ulimit -s {32 * 1024}; {numa_args} $NOOP_HOME/build/emu -i {workload} {emu_args} {fork_args} {diff_args} {chiseldb_args}')
270        return return_code
271
272    def run_simv(self, workload):
273        print("Running XiangShan simv with the following configurations:")
274        self.show()
275        diff_args = "$NOOP_HOME/"+ args.diff
276        assert_args = "-assert finish_maxfail=30 -assert global_finish_maxfail=10000"
277        return_code = self.__exec_cmd(f'cd $NOOP_HOME/build && ./simv +workload={workload} +diff={diff_args} +dump-wave=fsdb {assert_args} | tee simv.log')
278        with open(f"{self.args.noop_home}/build/simv.log") as f:
279            content = f.read()
280            if "Offending" in content or "HIT GOOD TRAP" not in content:
281                return 1
282        return return_code
283
284    def run(self, args):
285        if args.ci is not None:
286            return self.run_ci(args.ci)
287        if args.ci_vcs is not None:
288            return self.run_ci_vcs(args.ci_vcs)
289        actions = [
290            (args.generate, lambda _ : self.generate_verilog()),
291            (args.vcs_gen, lambda _ : self.generate_sim_verilog()),
292            (args.build, lambda _ : self.build_emu()),
293            (args.vcs_build, lambda _ : self.build_simv()),
294            (args.workload, lambda args: self.run_emu(args.workload)),
295            (args.clean, lambda _ : self.make_clean())
296        ]
297        valid_actions = map(lambda act: act[1], filter(lambda act: act[0], actions))
298        for i, action in enumerate(valid_actions):
299            print(f"Action {i}:")
300            ret = action(args)
301            if ret:
302                return ret
303        return 0
304
305    def __exec_cmd(self, cmd):
306        env = dict(os.environ)
307        env.update(self.args.get_env_variables())
308        print("subprocess call cmd:", cmd)
309        start = time.time()
310        proc = subprocess.Popen(cmd, shell=True, env=env, preexec_fn=os.setsid)
311        try:
312            return_code = proc.wait(self.timeout)
313            end = time.time()
314            print(f"Elapsed time: {end - start} seconds")
315            return return_code
316        except (KeyboardInterrupt, subprocess.TimeoutExpired):
317            os.killpg(os.getpgid(proc.pid), signal.SIGINT)
318            print(f"KeyboardInterrupt or TimeoutExpired.")
319            return 0
320
321    def __get_ci_cputest(self, name=None):
322        base_dir = os.path.join(self.args.am_home, "tests/cputest/build")
323        cputest = os.listdir(base_dir)
324        cputest = filter(lambda x: x.endswith(".bin"), cputest)
325        cputest = map(lambda x: os.path.join(base_dir, x), cputest)
326        return cputest
327
328    def __get_ci_rvtest(self, name=None):
329        base_dir = os.path.join(self.args.rvtest_home, "isa/build")
330        riscv_tests = os.listdir(base_dir)
331        riscv_tests = filter(lambda x: x.endswith(".bin"), riscv_tests)
332        all_rv_tests = ["rv64ui", "rv64um", "rv64ua", "rv64uf", "rv64ud"]
333        riscv_tests = filter(lambda x: x[:6] in all_rv_tests, riscv_tests)
334        riscv_tests = map(lambda x: os.path.join(base_dir, x), riscv_tests)
335        return riscv_tests
336
337    def __get_ci_misc(self, name=None):
338        base_dir = "/nfs/home/share/ci-workloads"
339        workloads = [
340            "bitmanip/bitMisc.bin",
341            "crypto/crypto-riscv64-noop.bin",
342            "coremark_rv64gc_o2/coremark-riscv64-xs.bin",
343            "coremark_rv64gc_o3/coremark-riscv64-xs.bin",
344            "coremark_rv64gcb_o3/coremark-riscv64-xs.bin",
345            "ext_intr/amtest-riscv64-xs.bin",
346            "cache-alias/aliastest-riscv64-xs.bin",
347            "Svinval/rv64mi-p-svinval.bin",
348            "pmp/pmp.riscv.bin",
349            "pmp/pmp-am/amtest-riscv64-xs.bin",
350            "pmp/hugepage-pmp-atom/amtest-riscv64-xs.bin",
351            "asid/asid.bin",
352            "isa_misc/xret_clear_mprv.bin",
353            "isa_misc/satp_ppn.bin",
354            "cache-management/softprefetchtest-riscv64-xs.bin"
355        ]
356        misc_tests = map(lambda x: os.path.join(base_dir, x), workloads)
357        return misc_tests
358
359    def __get_ci_mc(self, name=None):
360        base_dir = "/nfs/home/share/ci-workloads"
361        workloads = [
362            "dualcoretest/ldvio-riscv64-xs.bin"
363        ]
364        mc_tests = map(lambda x: os.path.join(base_dir, x), workloads)
365        return mc_tests
366
367    def __get_ci_nodiff(self, name=None):
368        base_dir = "/nfs/home/share/ci-workloads"
369        workloads = [
370            "cache-management/cacheoptest-riscv64-xs.bin"
371        ]
372        tests = map(lambda x: os.path.join(base_dir, x), workloads)
373        return tests
374
375    def __am_apps_path(self, bench):
376        filename = f"{bench}-riscv64-noop.bin"
377        return [os.path.join(self.args.am_home, "apps", bench, "build", filename)]
378
379    def __get_ci_workloads(self, name):
380        workloads = {
381            "linux-hello": "bbl.bin",
382            "linux-hello-smp": "bbl.bin",
383            "linux-hello-opensbi": "fw_payload.bin",
384            "linux-hello-smp-opensbi": "fw_payload.bin",
385            "linux-hello-new": "bbl.bin",
386            "linux-hello-smp-new": "bbl.bin",
387            "povray": "_700480000000_.gz",
388            "mcf": "_17520000000_.gz",
389            "xalancbmk": "_266100000000_.gz",
390            "gcc": "_39720000000_.gz",
391            "namd": "_434640000000_.gz",
392            "milc": "_103620000000_.gz",
393            "lbm": "_140840000000_.gz",
394            "gromacs": "_275480000000_.gz",
395            "wrf": "_1916220000000_.gz",
396            "astar": "_122060000000_.gz"
397        }
398        if name in workloads:
399            return [os.path.join("/nfs/home/share/ci-workloads", name, workloads[name])]
400        # select a random SPEC checkpoint
401        assert(name == "random")
402        all_cpt = [
403            "/nfs-nvme/home/share/checkpoints_profiles/spec06_rv64gcb_o2_20m/take_cpt",
404            "/nfs-nvme/home/share/checkpoints_profiles/spec06_rv64gcb_o3_20m/take_cpt",
405            "/nfs-nvme/home/share/checkpoints_profiles/spec06_rv64gc_o2_20m/take_cpt",
406            "/nfs-nvme/home/share/checkpoints_profiles/spec06_rv64gc_o2_50m/take_cpt",
407            "/nfs-nvme/home/share/checkpoints_profiles/spec17_rv64gcb_o2_20m/take_cpt",
408            "/nfs-nvme/home/share/checkpoints_profiles/spec17_rv64gcb_o3_20m/take_cpt",
409            "/nfs-nvme/home/share/checkpoints_profiles/spec17_rv64gc_o2_50m/take_cpt",
410            "/nfs-nvme/home/share/checkpoints_profiles/spec17_speed_rv64gcb_o3_20m/take_cpt"
411        ]
412        all_json = [
413            "/nfs-nvme/home/share/checkpoints_profiles/spec06_rv64gcb_o2_20m/json/simpoint_summary.json",
414            "/nfs-nvme/home/share/checkpoints_profiles/spec06_rv64gcb_o3_20m/simpoint_summary.json",
415            "/nfs-nvme/home/share/checkpoints_profiles/spec06_rv64gc_o2_20m/simpoint_summary.json",
416            "/nfs-nvme/home/share/checkpoints_profiles/spec06_rv64gc_o2_50m/simpoint_summary.json",
417            "/nfs-nvme/home/share/checkpoints_profiles/spec17_rv64gcb_o2_20m/simpoint_summary.json",
418            "/nfs-nvme/home/share/checkpoints_profiles/spec17_rv64gcb_o3_20m/simpoint_summary.json",
419            "/nfs-nvme/home/share/checkpoints_profiles/spec17_rv64gc_o2_50m/simpoint_summary.json",
420            "/nfs-nvme/home/share/checkpoints_profiles/spec17_speed_rv64gcb_o3_20m/simpoint_summary.json"
421        ]
422        assert(len(all_cpt) == len(all_json))
423        cpt_path, json_path = random.choice(list(zip(all_cpt, all_json)))
424        all_gcpt = load_all_gcpt(cpt_path, json_path)
425        return [random.choice(all_gcpt)]
426
427    def run_ci(self, test):
428        all_tests = {
429            "cputest": self.__get_ci_cputest,
430            "riscv-tests": self.__get_ci_rvtest,
431            "misc-tests": self.__get_ci_misc,
432            "mc-tests": self.__get_ci_mc,
433            "nodiff-tests": self.__get_ci_nodiff,
434            "microbench": self.__am_apps_path,
435            "coremark": self.__am_apps_path
436        }
437        for target in all_tests.get(test, self.__get_ci_workloads)(test):
438            print(target)
439            ret = self.run_emu(target)
440            if ret:
441                if self.args.default_wave_home != self.args.wave_home:
442                    print("copy wave file to " + self.args.wave_home)
443                    self.__exec_cmd(f"cp $NOOP_HOME/build/*.vcd $WAVE_HOME")
444                    self.__exec_cmd(f"cp $NOOP_HOME/build/emu $WAVE_HOME")
445                    self.__exec_cmd(f"cp $NOOP_HOME/build/rtl/SimTop.v $WAVE_HOME")
446                    self.__exec_cmd(f"cp $NOOP_HOME/build/*.db $WAVE_HOME")
447                return ret
448        return 0
449
450    def run_ci_vcs(self, test):
451        all_tests = {
452            "cputest": self.__get_ci_cputest,
453            "riscv-tests": self.__get_ci_rvtest,
454            "misc-tests": self.__get_ci_misc,
455            "mc-tests": self.__get_ci_mc,
456            "nodiff-tests": self.__get_ci_nodiff,
457            "microbench": self.__am_apps_path,
458            "coremark": self.__am_apps_path
459        }
460        for target in all_tests.get(test, self.__get_ci_workloads)(test):
461            print(target)
462            ret = self.run_simv(target)
463            if ret:
464                if self.args.default_wave_home != self.args.wave_home:
465                    print("copy wave file to " + self.args.wave_home)
466                    self.__exec_cmd(f"cp $NOOP_HOME/build/*.fsdb $WAVE_HOME")
467                    self.__exec_cmd(f"cp $NOOP_HOME/build/simv $WAVE_HOME")
468                    self.__exec_cmd(f"cp $NOOP_HOME/build/rtl/SimTop.v $WAVE_HOME")
469                    self.__exec_cmd(f"cp $NOOP_HOME/build/*.db $WAVE_HOME")
470                return ret
471        return 0
472
473def get_free_cores(n):
474    while True:
475        # To avoid potential conflicts, we allow CI to use SMT.
476        num_logical_core = psutil.cpu_count(logical=False)
477        core_usage = psutil.cpu_percent(interval=1, percpu=True)
478        num_window = num_logical_core // n
479        for i in range(num_window):
480            window_usage = core_usage[i * n : i * n + n]
481            if sum(window_usage) < 30 * n and True not in map(lambda x: x > 90, window_usage):
482                return (((i * n) % 128)// 64, i * n, i * n + n - 1)
483        print(f"No free {n} cores found. CPU usage: {core_usage}\n")
484
485if __name__ == "__main__":
486    parser = argparse.ArgumentParser(description='Python wrapper for XiangShan')
487    parser.add_argument('workload', nargs='?', type=str, default="",
488                        help='input workload file in binary format')
489    # actions
490    parser.add_argument('--build', action='store_true', help='build XS emu')
491    parser.add_argument('--generate', action='store_true', help='generate XS verilog')
492    parser.add_argument('--vcs-gen', action='store_true', help='generate XS sim verilog for vcs')
493    parser.add_argument('--vcs-build', action='store_true', help='build XS simv')
494    parser.add_argument('--ci', nargs='?', type=str, const="", help='run CI tests')
495    parser.add_argument('--ci-vcs', nargs='?', type=str, const="", help='run CI tests on simv')
496    parser.add_argument('--clean', action='store_true', help='clean up XiangShan CI workspace')
497    parser.add_argument('--timeout', nargs='?', type=int, default=None, help='timeout (in seconds)')
498    # environment variables
499    parser.add_argument('--nemu', nargs='?', type=str, help='path to nemu')
500    parser.add_argument('--am', nargs='?', type=str, help='path to nexus-am')
501    parser.add_argument('--dramsim3', nargs='?', type=str, help='path to dramsim3')
502    parser.add_argument('--rvtest', nargs='?', type=str, help='path to riscv-tests')
503    parser.add_argument('--wave-dump', nargs='?', type=str , help='path to dump wave')
504    # chisel arguments
505    parser.add_argument('--enable-log', action='store_true', help='enable log')
506    parser.add_argument('--num-cores', type=int, help='number of cores')
507    # makefile arguments
508    parser.add_argument('--release', action='store_true', help='enable release')
509    parser.add_argument('--spike', action='store_true', help='enable spike diff')
510    parser.add_argument('--with-dramsim3', action='store_true', help='enable dramsim3')
511    parser.add_argument('--threads', nargs='?', type=int, help='number of emu threads')
512    parser.add_argument('--trace', action='store_true', help='enable vcd waveform')
513    parser.add_argument('--trace-fst', action='store_true', help='enable fst waveform')
514    parser.add_argument('--config', nargs='?', type=str, help='config')
515    parser.add_argument('--mfc', action='store_true', help='use mfc')
516    parser.add_argument('--emu-optimize', nargs='?', type=str, help='verilator optimization letter')
517    parser.add_argument('--xprop', action='store_true', help='enable xprop for vcs')
518    # emu arguments
519    parser.add_argument('--numa', action='store_true', help='use numactl')
520    parser.add_argument('--diff', nargs='?', default="./ready-to-run/riscv64-nemu-interpreter-so", type=str, help='nemu so')
521    parser.add_argument('--max-instr', nargs='?', type=int, help='max instr')
522    parser.add_argument('--disable-fork', action='store_true', help='disable lightSSS')
523    parser.add_argument('--no-diff', action='store_true', help='disable difftest')
524    parser.add_argument('--ram-size', nargs='?', type=str, help='manually set simulation memory size (8GB by default)')
525    # both makefile and emu arguments
526    parser.add_argument('--no-db', action='store_true', help='disable chiseldb dump')
527    parser.add_argument('--pgo', nargs='?', type=str, help='workload for pgo (null to disable pgo)')
528    parser.add_argument('--pgo-max-cycle', nargs='?', default=400000, type=int, help='maximun cycle to train pgo')
529    parser.add_argument('--pgo-emu-args', nargs='?', default='--no-diff', type=str, help='emu arguments for pgo')
530    parser.add_argument('--llvm-profdata', nargs='?', type=str, help='corresponding llvm-profdata command of clang to compile emu, do not set with GCC')
531
532    args = parser.parse_args()
533
534    xs = XiangShan(args)
535    ret = xs.run(args)
536
537    sys.exit(ret)
538