1 2import sys 3import argparse 4import yaml 5import os 6from datetime import date 7import time 8 9github_yaml = "" 10workspace = "" 11nemu_home = "" 12am_home = "" 13head_sha = "" 14wave_home = "" 15perf_home = "" 16set_env = "" 17sh_path = "" 18 19def remove_empty(list_str): 20 list_new = [] 21 for s in list_str: 22 if not (s == ""): 23 list_new.append(s) 24 return list_new 25 26def split_cmd(cmd): 27 c = cmd.replace("\\\n", " ") 28 c = remove_empty(c.split("\n")) 29 cs = [] 30 for ci in c: 31 ci = ci.replace("\\\n", " ") 32 ci = ci.replace("\n\n", "\n") 33 ci = ci.replace(" ", " ") 34 if ci[-1] == "\n": 35 ci = ci[:-1] 36 cs.append(ci) 37 return cs 38 39def parse_yaml(yaml_file): 40 file_data = open(yaml_file, "r", encoding="utf-8").read() 41 yaml_data = yaml.load(file_data, Loader=yaml.CLoader) 42 return yaml_data 43 44def show(test_info): 45 name = test_info["name"] 46 print(name) 47 48def run_test(test_info, numa, run_mode): 49 name = test_info["name"] 50 coe_key = "continue-on-error" 51 to_key = "timeout-minutes" 52 s_key = "steps" 53 continue_on_error = True if (coe_key not in test_info.keys()) else (False if (test_info[coe_key] == "false") else True) 54 timeout_minutes = 9999 if (to_key not in test_info.keys()) else int(test_info[to_key]) 55 if s_key not in test_info.keys(): 56 print(name, " ", s_key, " not found in yaml, skip") 57 sys.exit() 58 steps_raw = test_info[s_key] 59 # print("Steps") 60 # print(steps) 61 steps = {} 62 for s in steps_raw: 63 if "name" in s.keys(): 64 steps[s["name"]] = s["run"] 65 # print(steps) 66 67 replace_list = [ 68 ["--numa", "--numa" if numa else ""], 69 ["$GITHUB_WORKSPACE", f"{workspace}"], 70 ["$HEAD_SHA", f"{head_sha}"], 71 ["$PERF_HOME", f"{perf_home}"], 72 ["$WAVE_HOME", f"{wave_home}"], 73 ["$AM_HOME", f"{am_home}"] 74 ] 75 76 steps.pop("set env") 77 78 for s in steps.keys(): 79 for r in replace_list: 80 steps[s] = steps[s].replace(r[0], r[1]) 81 82 cmd = [] 83 for s in steps.keys(): 84 cs = split_cmd(steps[s]) 85 cmd = cmd + cs 86 87 if run_mode: 88 os.system(f"mkdir -p {wave_home}\n") 89 os.system(f"mkdir -p {perf_home}\n") 90 for c in cmd: 91 f_c = set_env + " " + c 92 print(f"[CMD] {f_c}", flush=True) 93 os.system(f_c) 94 95 else: 96 if (sh_path is None): 97 print("sh_path is None") 98 sys.exit() 99 elif (not os.path.exists(sh_path)): 100 os.mkdir(sh_path) 101 102 103 sh_file_name = os.path.join(sh_path, "_".join(name.strip().replace("-", "").split())+".sh") 104 with open(sh_file_name, "w") as tmp_sh: 105 tmp_sh.write(f"mkdir -p {wave_home}\n") 106 tmp_sh.write(f"mkdir -p {perf_home}\n") 107 for c in cmd: 108 print(c) 109 tmp_sh.write(c+"\n") 110 111if __name__ == "__main__": 112 # Usage: 113 # 1. run ci test 114 # python3 scripts/local_ci.py --xs-path $(pwd) --run 115 # More Params: 116 # --pick-test MC: only run 'EMU - MC' 117 118 # 2. print ci test name 119 # python3 scripts/local_ci.py --xs-path $(pwd) --show-test 120 # This can also use --pick-test 121 122 # 3. print ci test command into splited sh files. Run the sh manualy. 123 # python3 scripts/local_ci.py --xs-path $(pwd) --sh-path /nfs/home/zhangzifei/work/xs-master/ci-sh 124 # just remove --run 125 126 # Other Params: 127 # --numa: use numa ctrl, require eypc 128 # --head-sha: magic word, default is today's date 129 # --nemu-home/--am-home: don't know if it is used 130 131 parser = argparse.ArgumentParser(description="run ci locally") 132 parser.add_argument("--xs-path", type=str, help="XiangShan, NOOP_HOME") 133 parser.add_argument("--nemu-home", type=str, help="NEMU_HOME") 134 parser.add_argument("--am-home", type=str, help="AM_HOME") 135 parser.add_argument("--sh-path", type=str, help="ci's sh file path") 136 parser.add_argument("--head-sha", type=str, help="magic word") 137 parser.add_argument("--run", action='store_true', help="run test, not gen sh") 138 parser.add_argument("--numa", action='store_true', help="epyc numactl") 139 parser.add_argument("--show-test", action="store_true", help="print test case") 140 parser.add_argument("--pick-test", type=str, help="pick only one test") 141 142 args = parser.parse_args() 143 144 print(args) 145 146 workspace = os.getenv("NOOP_HOME") if (args.xs_path is None) else args.xs_path 147 head_sha = date.today().strftime(r"%Y%m%d") if (args.head_sha is None) else args.head_sha 148 wave_home = os.path.join(workspace, "wave", head_sha) 149 perf_home = os.path.join(workspace, "perf", head_sha) 150 github_yaml = os.path.join(workspace, ".github/workflows/emu.yml") 151 nemu_home = os.getenv("NEMU_HOME") if (args.nemu_home is None) else args.nemu_home 152 am_home = os.getenv("AM_HOME") if (args.am_home is None) else args.am_home 153 set_env = f"NEMU_HOME={nemu_home} NOOP_HOME={workspace} WAVE_HOME={wave_home} PERF_HOME={perf_home} AM_HOME={am_home}" 154 sh_path = f"{workspace}/ci-sh" if (args.sh_path is None) else args.sh_path 155 156 print("workspace(NOOP_HOME): ", workspace) 157 print("head_sha: ", head_sha) 158 print("wave_home: ", wave_home) 159 print("perf_home: ", perf_home) 160 print("github_yaml: ", github_yaml) 161 print("nemu_home: ", nemu_home) 162 print("am_home: ", am_home) 163 print("set_env: ", set_env) 164 print("sh_path", sh_path) 165 166 input("Press Enter to continue\n") 167 168 ci_tests = parse_yaml(github_yaml)["jobs"] 169 170 if (args.show_test): 171 for test in ci_tests.keys(): 172 show(ci_tests[test]) 173 else: 174 for test in ci_tests.keys(): 175 if args.pick_test is not None: 176 if (args.pick_test in ci_tests[test]["name"]): 177 run_test(ci_tests[test], args.numa, args.run) 178 else: 179 run_test(ci_tests[test], args.numa, args.run) 180