Lines Matching +full:test +full:. +full:rgb
9 "# benchmark.ipynb\n",
10 "# Part of the aflplusplus project, requires an ipynb (Jupyter) editor or viewer.\n",
11 "# Author: Chris Ball <chris@printf.net>\n",
14 "with open(\"benchmark-results.jsonl\") as f:\n",
15 " lines = f.read().splitlines()\n",
16 "json_lines = [json.loads(line) for line in lines]\n"
25 "We have JSON Lines in [benchmark-results.jsonl](benchmark-results.jsonl) that look like this:"
43 " \"compiler\": \"clang version 16.0.6\",\n",
52 " \"test-instr\": {\n",
65 "print(json.dumps(json.loads(lines[0]), indent=2))"
72 …"The [pd.json_normalize()](https://pandas.pydata.org/docs/reference/api/pandas.json_normalize.html…
85 " .dataframe tbody tr th:only-of-type {\n",
89 " .dataframe tbody tr th {\n",
93 " .dataframe thead th {\n",
101 " <th>config.afl_persistent_config</th>\n",
102 " <th>config.afl_system_config</th>\n",
103 " <th>config.afl_version</th>\n",
104 " <th>config.comment</th>\n",
105 " <th>config.compiler</th>\n",
106 " <th>config.target_arch</th>\n",
107 " <th>hardware.cpu_fastest_core_mhz</th>\n",
108 " <th>hardware.cpu_model</th>\n",
109 " <th>hardware.cpu_threads</th>\n",
110 " <th>targets.test-instr.singlecore.execs_per_sec</th>\n",
111 " <th>...</th>\n",
112 " <th>targets.test-instr.singlecore.fuzzers_used</th>\n",
113 " <th>targets.test-instr-persist-shmem.singlecore.execs_per_sec</th>\n",
114 " <th>targets.test-instr-persist-shmem.singlecore.execs_total</th>\n",
115 " <th>targets.test-instr-persist-shmem.singlecore.fuzzers_used</th>\n",
116 " <th>targets.test-instr-persist-shmem.multicore.execs_per_sec</th>\n",
117 " <th>targets.test-instr-persist-shmem.multicore.execs_total</th>\n",
118 " <th>targets.test-instr-persist-shmem.multicore.fuzzers_used</th>\n",
119 " <th>targets.test-instr.multicore.execs_per_sec</th>\n",
120 " <th>targets.test-instr.multicore.execs_total</th>\n",
121 " <th>targets.test-instr.multicore.fuzzers_used</th>\n",
131 " <td>clang version 16.0.6</td>\n",
137 " <td>...</td>\n",
155 " <td>clang version 16.0.6</td>\n",
161 " <td>...</td>\n",
179 " <td>clang version 16.0.6</td>\n",
185 " <td>...</td>\n",
203 " <td>clang version 16.0.6</td>\n",
209 " <td>...</td>\n",
227 " <td>clang version 16.0.6</td>\n",
233 " <td>...</td>\n",
251 " config.afl_persistent_config config.afl_system_config config.afl_version \\\n",
258 " config.comment config.compiler \\\n",
259 "0 i9-9900k, 16GB DDR4-3000, Arch Linux clang version 16.0.6 \n",
260 "1 i9-9900k, 16GB DDR4-3000, Arch Linux clang version 16.0.6 \n",
261 "2 i9-9900k, 16GB DDR4-3000, Arch Linux clang version 16.0.6 \n",
262 "3 i9-9900k, 16GB DDR4-3000, Arch Linux clang version 16.0.6 \n",
263 "4 i9-9900k, 16GB DDR4-3000, Arch Linux clang version 16.0.6 \n",
265 " config.target_arch hardware.cpu_fastest_core_mhz \\\n",
272 " hardware.cpu_model hardware.cpu_threads \\\n",
279 " targets.test-instr.singlecore.execs_per_sec ... \\\n",
280 "0 9845.64 ... \n",
281 "1 NaN ... \n",
282 "2 NaN ... \n",
283 "3 NaN ... \n",
284 "4 NaN ... \n",
286 " targets.test-instr.singlecore.fuzzers_used \\\n",
293 " targets.test-instr-persist-shmem.singlecore.execs_per_sec \\\n",
300 " targets.test-instr-persist-shmem.singlecore.execs_total \\\n",
307 " targets.test-instr-persist-shmem.singlecore.fuzzers_used \\\n",
314 " targets.test-instr-persist-shmem.multicore.execs_per_sec \\\n",
321 " targets.test-instr-persist-shmem.multicore.execs_total \\\n",
328 " targets.test-instr-persist-shmem.multicore.fuzzers_used \\\n",
335 " targets.test-instr.multicore.execs_per_sec \\\n",
342 " targets.test-instr.multicore.execs_total \\\n",
349 " targets.test-instr.multicore.fuzzers_used \n",
366 "df = pd.json_normalize(json_lines)\n",
367 "df.head()"
376 …in each config setting -- where the x-axis is number of cores, and the y-axis is execs_per_sec.\n",
387 "i7 = df.query(\"`config.comment` == 'i9-9900k, 16GB DDR4-3000, Arch Linux'\")\n",
397 "def build_graphdf_from_query(query: pd.DataFrame):\n",
398 " \"\"\"Build a table suitable for graphing from a subset of the dataframe.\"\"\"\n",
400 …int(query[[\"targets.test-instr-persist-shmem.multicore.fuzzers_used\", \"targets.test-instr.multi…
401 " for _, row in query.iterrows():\n",
402 " for target in [\"test-instr-persist-shmem\", \"test-instr\"]:\n",
405 " if not row[f\"targets.{target}.{mode}.execs_per_sec\"] > 0:\n",
407 " execs_per_sec = row[f\"targets.{target}.{mode}.execs_per_sec\"]\n",
408 " parallel_fuzzers = row[f\"targets.{target}.{mode}.fuzzers_used\"]\n",
409 " afl_persistent_config = row[\"config.afl_persistent_config\"]\n",
410 " afl_system_config = row[\"config.afl_system_config\"]\n",
411 " if target == \"test-instr-persist-shmem\":\n",
425 …" graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": par…
427 …" graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": par…
429 …" graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": par…
432 …" graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\":…
435 …" graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\":…
436 " return pd.DataFrame.from_records(graphdata).sort_values(\"label\", ascending=False)\n",
449 ….w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"1200\" height=\"400\" styl…
458 "pd.options.plotting.backend = \"plotly\"\n",
462 …"pivotdf = graphdf.pivot(index=\"parallel_fuzzers\", columns=\"label\", values=\"execs_per_sec\")\…
463 "fig = pivotdf.plot(\n",
473 "tickvals = np.linspace(graphdf['execs_per_sec'].min(), graphdf['execs_per_sec'].max(), 6)\n",
474 "ticktext = [f\"{val:.0f}x\" for val in tickvals / graphdf['execs_per_sec'].min()]\n",
476 "fig.update_yaxes(tickvals=tickvals, ticktext=ticktext)\n",
477 "fig.update_xaxes(tickvals=list(range(1,36+1)))\n",
478 "fig.update_layout(width=1200, height=400)\n",
479 "fig.show(\"svg\")\n"
499 " .dataframe tbody tr th:only-of-type {\n",
503 " .dataframe tbody tr th {\n",
507 " .dataframe thead th {\n",
1033 …. It's just preparing Markdown for the block below it to render. Jupyter Notebooks aren't able t…
1046 …ere are a few things that jump out from the graph above. Let's start at the bottom of the graph.\…
1048 "#### test-instr vs. test-instr-persist-shmem\n",
1051 …"represented as **1.0x**. If you build and run a fuzzer without creating a persistent mode harnes…
1052 "you get on this machine.\n",
1054 "#### Multicore test-instr\n",
1056 …here are CPU threads, we can reach 101692 execs per second, which is **10.3x** that base speed.\n",
1062 …y as described [here](https://github.com/AFLplusplus/AFLplusplus/blob/stable/instrumentation/READM…
1063 …"we end up with **12.8x** base speed. So -- perhaps counter-intuively -- if you have a choice bet…
1064 …machine) to use persistent mode on a single core, than to use non-persistent mode on all cores.\n",
1069 "**86.1x** base speed.\n",
1074 …"Meltdown, Retbleed, etc) introduced in Linux v5.2. Disabling these results in a `execs_per_sec` …
1075 "118.7x (mitigations off) and 86.1x (mitigations on) base speed. Turning on mitigations\n",
1079 "number of execs reached with 7 threads and mitigations off.\n",
1082 "mitigations is 321386 execs per sec, which is the averaged performance of 2.9 cores.\n",
1085 "than where we started from.\n",
1089 "* Using >16 is worse than using 16. Makes sense.\n",
1090 …se the number of CPUs in /proc/cpuinfo (threads) to get the best performance. But if we did halve…
1092 " of performance. This could be a good tradeoff in terms of cost.\n"
1095 "<IPython.core.display.Markdown object>"
1104 "# (You can ignore reading this code cell.)\n",
1105 "from IPython.display import Markdown as md\n",
1106 … "singlecore_base_execs = pivotdf.iloc[0][\"Singlecore: Non-persistent mode + kernel config\"]\n",
1107 …"singlecore_persist_execs = pivotdf.iloc[0][\"Singlecore: Persistent mode/shared memory + kernel c…
1108 …max_execs = int(pivotdf[\"Multicore: Persistent mode/shared memory + kernel config\"].idxmax())\n",
1109 … "multicore_base_max_execs = pivotdf[\"Multicore: Non-persistent mode + kernel config\"].max()\n",
1113 …"multicore_max_execs_mitigations_off = pivotdf[multicore_persistent_without_mitigations_label].max…
1114 …"multicore_max_execs_mitigations_off_only_cores = pivotdf.loc[multicore_fuzzers_max_execs / 2][mul…
1115 …igations_on = pivotdf[\"Multicore: Persistent mode/shared memory without kernel config\"].max()\n",
1116 …g_gain_per_core = pivotdf.loc[pivotdf.index <= 8][\"Multicore: Persistent mode/shared memory + ker…
1121 …ere are a few things that jump out from the graph above. Let's start at the bottom of the graph.\…
1123 "#### test-instr vs. test-instr-persist-shmem\n",
1126 …"represented as **1.0x**. If you build and run a fuzzer without creating a persistent mode harnes…
1127 "you get on this machine.\n",
1129 "#### Multicore test-instr\n",
1131 …} execs per second, which is **{factor_for_execs(multicore_base_max_execs)}x** that base speed.\n",
1137 …y as described [here](https://github.com/AFLplusplus/AFLplusplus/blob/stable/instrumentation/READM…
1138 …"we end up with **{factor_for_execs(singlecore_persist_execs)}x** base speed. So -- perhaps count…
1139 …machine) to use persistent mode on a single core, than to use non-persistent mode on all cores.\n",
1144 "**{factor_for_execs(multicore_max_execs_mitigations_on)}x** base speed.\n",
1149 …"Meltdown, Retbleed, etc) introduced in Linux v5.2. Disabling these results in a `execs_per_sec` …
1150 …_for_execs(multicore_max_execs_mitigations_on)}x (mitigations on) base speed. Turning on mitigati…
1154 "number of execs reached with 7 threads and mitigations off.\n",
1157 …eraged performance of {round(mitigations_off_increase / multicore_avg_gain_per_core, 1)} cores.\n",
1160 "than where we started from.\n",
1164 "* Using >16 is worse than using 16. Makes sense.\n",
1165 …"* So, we should use the number of CPUs in /proc/cpuinfo (threads) to get the best performance. B…
1167 " of performance. This could be a good tradeoff in terms of cost.\n",
1177 …f CPU threads you have access to. What if there were more threads? Here the experiment is repeat…
1190 " .dataframe tbody tr th:only-of-type {\n",
1194 " .dataframe tbody tr th {\n",
1198 " .dataframe thead th {\n",
1206 " <th>config.afl_persistent_config</th>\n",
1207 " <th>config.afl_system_config</th>\n",
1208 " <th>config.afl_version</th>\n",
1209 " <th>config.comment</th>\n",
1210 " <th>config.compiler</th>\n",
1211 " <th>config.target_arch</th>\n",
1212 " <th>hardware.cpu_fastest_core_mhz</th>\n",
1213 " <th>hardware.cpu_model</th>\n",
1214 " <th>hardware.cpu_threads</th>\n",
1215 " <th>targets.test-instr-persist-shmem.multicore.execs_per_sec</th>\n",
1216 " <th>targets.test-instr-persist-shmem.multicore.execs_total</th>\n",
1217 " <th>targets.test-instr-persist-shmem.multicore.fuzzers_used</th>\n",
1226 " <td>AWS EC2 r6a.48xlarge spot instance</td>\n",
1227 " <td>clang version 15.0.7 (Amazon Linux 15.0.7-3.am...</td>\n",
1241 " <td>AWS EC2 r6a.48xlarge spot instance</td>\n",
1242 " <td>clang version 15.0.7 (Amazon Linux 15.0.7-3.am...</td>\n",
1256 " config.afl_persistent_config config.afl_system_config \\\n",
1260 " config.afl_version config.comment \\\n",
1261 "223 ++4.09a AWS EC2 r6a.48xlarge spot instance \n",
1262 "224 ++4.09a AWS EC2 r6a.48xlarge spot instance \n",
1264 " config.compiler \\\n",
1265 "223 clang version 15.0.7 (Amazon Linux 15.0.7-3.am... \n",
1266 "224 clang version 15.0.7 (Amazon Linux 15.0.7-3.am... \n",
1268 " config.target_arch hardware.cpu_fastest_core_mhz \\\n",
1272 " hardware.cpu_model hardware.cpu_threads \\\n",
1276 " targets.test-instr-persist-shmem.multicore.execs_per_sec \\\n",
1280 " targets.test-instr-persist-shmem.multicore.execs_total \\\n",
1284 " targets.test-instr-persist-shmem.multicore.fuzzers_used \n",
1295 "r6a = df.query(\"`config.comment` == 'AWS EC2 r6a.48xlarge spot instance'\")\n",
1296 "r6a.head(2).dropna(axis=1)"
1309 " .dataframe tbody tr th:only-of-type {\n",
1313 " .dataframe tbody tr th {\n",
1317 " .dataframe thead th {\n",
1339 " <td>Multicore: Persistent mode/shared memory + ker...</td>\n",
1347 " <td>Multicore: Persistent mode/shared memory + ker...</td>\n",
1359 "0 Multicore: Persistent mode/shared memory + ker... \n",
1360 "1 Multicore: Persistent mode/shared memory + ker... "
1370 "r6a_graphdf.head(2)"
1381 ….w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"1200\" height=\"400\" styl…
1389 …"r6a_pivotdf = r6a_graphdf.pivot(index=\"parallel_fuzzers\", columns=\"label\", values=\"execs_per…
1390 "r6a_fig = r6a_pivotdf.plot(\n",
1400 …"tickvals = np.linspace(r6a_graphdf['execs_per_sec'].min(), r6a_graphdf['execs_per_sec'].max(), 6)…
1401 "ticktext = [f\"{val:.0f}x\" for val in tickvals / graphdf['execs_per_sec'].min()]\n",
1403 "r6a_fig.update_yaxes(tickvals=tickvals, ticktext=ticktext)\n",
1404 "r6a_fig.update_xaxes(tickvals=list(range(0,200+1, 4)))\n",
1405 "r6a_fig.update_layout(width=1200, height=400)\n",
1406 "r6a_fig.show(\"svg\")"
1415 …sing 8 fuzzers. Using 192 parallel fuzzers (the physical number of threads in this machine) gives…
1417 …measuring before simply using the number of hardware threads in your machine. But does this mean …
1419 …. A good way to test this theory would be to run more system-call-servicers (read: kernels!) at o…
1434 "file_extension": ".py",
1439 "version": "3.11.5"