1# Copyright (c) 2019-2021 Arm Limited. 2# 3# SPDX-License-Identifier: MIT 4# 5# Permission is hereby granted, free of charge, to any person obtaining a copy 6# of this software and associated documentation files (the "Software"), to 7# deal in the Software without restriction, including without limitation the 8# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 9# sell copies of the Software, and to permit persons to whom the Software is 10# furnished to do so, subject to the following conditions: 11# 12# The above copyright notice and this permission notice shall be included in all 13# copies or substantial portions of the Software. 14# 15# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21# SOFTWARE. 22 23#!/bin/sh 24 25# Global: Global variables and global settings {{{ 26# Treat unset variables as an error when substituting 27set -u 28 29CMD=$( basename $0 ) 30 31# All supported strategy options 32ALL_STRATEGY_OPTIONS=("native" "reshaped_rhs_only" "reshaped") 33 34# All supported data type options 35ALL_DATA_TYPE_OPTIONS=("f32" "f16" "qasymm8") 36 37# Names of example binary for each strategy 38EXAMPLE_BIN_NATIVE="benchmark_cl_gemm_native" 39EXAMPLE_BIN_RESHAPED_RHS_ONLY="benchmark_cl_gemm_reshaped_rhs_only" 40EXAMPLE_BIN_RESHAPED="benchmark_cl_gemm_reshaped" 41EXAMPLE_BIN_RESHAPED_RHS_ONLY_LOWP="benchmark_cl_gemmlowp_reshaped_rhs_only_fused_output_stage_fixedpoint" 42EXAMPLE_BIN_RESHAPED_LOWP="benchmark_cl_gemmlowp_reshaped" 43 44# Default data type 45DEFAULT_DATA_TYPE="f32" 46 47# Default output directory 48DEFAULT_OUT_DIR="out" 49 50# Default ID of the first experiment 51DEFAULT_ID_EXPERIMENT_START=0 52 53# Default total number of experiments 54DEFAULT_NUM_EXPERIMENTS="all" 55 56# Default output file extension 57DEFAULT_OUT_EXTENSION="mlgo_benchmark" 58 59# Default OpenCL tuner mode 60DEFAULT_TUNER_MODE="rapid" 61 62# Number of iterations for each benchmark run 63NUM_ITERATION=5 64# Global }}} 65 66# Functions {{{ 67####################################### 68# Print gemm shape file help message 69# Globals: 70# None 71# Arguments: 72# None 73# Returns: 74# None 75####################################### 76function help_gemm_shape_file() { 77 cat >&2 << EOF 78Gemm shape file: 79 Gemm shape file is a csv file with fields separated by commas. The optional header and comments are ignored by the parser. 80 81 A gemm shape is a list of 4 positive integers <M, N, K, B> describing the shapes of the two matrices (LHS and RHS) 82 with: 83 M - Number of lhs matrix rows 84 N - Number of rhs matrix columns 85 K - Number of lhs matrix columns/rhs matrix rows 86 B - Batch size 87 88 An example gemm shape file looks like: 89 100,100,30,1 90 100,100,30,3 91 ... 92 93EOF 94} 95 96####################################### 97# Print gemm config file for native help message 98# Globals: 99# None 100# Arguments: 101# None 102# Returns: 103# None 104####################################### 105function help_gemm_config_file_native() { 106 cat >&2 << EOF 107Gemm config file (Strategy native): 108 Gemm config file is a csv file with fields separated by commas. The optional header and comments are ignored by the parser. 109 110 A gemm config is a list of 3 positive integers <m0, n0, k0>, with: 111 m0 - Number of rows processed by the matrix multiplication 112 n0 - Number of columns processed by the matrix multiplication 113 k0 - Number of partial accumulations performed by the matrix multiplication 114 115 Only the following configurations of M0, N0 and K0 are currently supported: 116 M0 = 1, 2, 3, 4, 5, 6, 7, 8 117 N0 = 2, 3, 4, 8, 16 118 K0 = 2, 3, 4, 8, 16 119 120 An example gemm config file looks like: 121 1,4,4 122 2,3,8 123 ... 124 125EOF 126} 127 128####################################### 129# Print gemm config file for reshaped_rhs_only help message 130# Globals: 131# None 132# Arguments: 133# None 134# Returns: 135# None 136####################################### 137function help_gemm_config_file_reshaped_rhs_only() { 138 cat >&2 << EOF 139Gemm config file (Strategy reshaped_rhs_only): 140 Gemm config file is a csv file with fields separated by commas. The optional header and comments are ignored by the parser. 141 142 A gemm config is a list of 4 positive integers <m0, n0, k0, h0> and 3 boolean values: 143 m0 - Number of rows processed by the matrix multiplication 144 n0 - Number of columns processed by the matrix multiplication 145 k0 - Number of partial accumulations performed by the matrix multiplication 146 h0 - Number of horizontal blocks of size (k0xn0) stored on the same output row 147 interleave_rhs - Interleave rhs matrix (1) / Do not interleave rhs matrix (0) 148 transpose_rhs - Transpose rhs matrix (1) / Do not transpose rhs matrix (0) 149 export_to_cl_image_rhs - (Not supported for quantized types) Export rhs matrix to cl_image (1) / Do not export rhs matrix to cl_image (0). Can only be true 150 with certain combinations of the GEMMParams and other configs. Please refer to CLGEMMReshapeRHSMatrixKernel 151 for more details 152 153 Only the following configurations of M0, N0 and K0 are currently supported: 154 M0 = 1, 2, 3, 4, 5, 6, 7, 8 155 N0 = 2, 3, 4, 8, 16 156 K0 = 2, 3, 4, 8, 16 157 H0 >= 1 158 159 An example gemm config file looks like: 160 4,4,4,1,1,1,0 161 4,4,4,3,1,0,1 162 ... 163 164EOF 165} 166 167####################################### 168# Print gemm config file for reshaped help message 169# Globals: 170# None 171# Arguments: 172# None 173# Returns: 174# None 175####################################### 176function help_gemm_config_file_reshaped() { 177 cat >&2 << EOF 178Gemm config file (Strategy reshaped): 179 Gemm config file is a csv file with fields separated by commas. The header and comments are ignored by the parser. 180 181 A gemm config is a list of 5 positive integers <m0, n0, k0, v0, h0> and 4 boolean values: 182 m0 - Number of rows processed by the matrix multiplication 183 n0 - Number of columns processed by the matrix multiplication 184 k0 - Number of partial accumulations performed by the matrix multiplication 185 v0 - Number of vertical blocks of size (m0xk0) stored on the same output row 186 h0 - Number of horizontal blocks of size (k0xn0) stored on the same output row 187 interleave_lhs - Interleave lhs matrix (1) / Do not interleave lhs matrix (0) 188 interleave_rhs - Interleave rhs matrix (1) / Do not interleave rhs matrix (0) 189 transpose_rhs - Transpose rhs matrix but not lhs matrix (1) / Do not transpose rhs matrix but do transpose lhs matrix (0) 190 export_to_cl_image_rhs - (Not supported for quantized types) Export rhs matrix to cl_image (1) / Do not export rhs matrix to cl_image (0). Can only be true 191 with certain combinations of the GEMMParams and other configs. Please refer to CLGEMMReshapeRHSMatrixKernel 192 for more details 193 194 If rhs matrix is transposed only the following configurations are currently supported: 195 M0 = 2, 3, 4, 5, 6, 7, 8 196 N0 = 2, 3, 4, 8, 16 197 K0 = 2, 3, 4, 8, 16 198 V0 >= 1 199 H0 >= 1 200 201 If lhs matrix is transposed only the following configurations are currently supported: 202 M0 = 2, 3, 4, 8 203 N0 = 2, 3, 4, 8, 16 204 K0 = 2, 3, 4, 8, 16 205 V0 >= 1 206 H0 >= 1 207 208 An example gemm config file looks like: 209 4,4,4,1,3,1,1,1,0 210 4,4,4,3,3,1,1,0,1 211 ... 212 213EOF 214} 215 216####################################### 217# Print usage of this program and exit with Error 218# Globals: 219# Assumes all globals are required 220# Arguments: 221# None 222# Returns: 223# Error(1) 224####################################### 225function usage() { 226 cat >&2 << EOF 227Run gemm examples of a selected strategy, over provided tunable configurationsa and gemm shapes. 228Save the benchmark results to json files in an output directory. 229 230Usage: ${CMD} [-h] -s <strategy> -e <example_binary_dir> -g <gemm_shape_file> -c <gemm_config_file> [-o <out_dir>] [-d <data_type>] [-i <id_experiment_start>] [-n <num_experiments>] [-t <output_extension>] 231 232Options: 233 -h 234 Print help messages. If a strategy is specified with -s <strategy>, then only display messages relevant to that 235 strategy. Otherwise if no strategy is specified, display messages for all available strategies. 236 237 -s <strategy> 238 Strategy option. 239 Options: ${ALL_STRATEGY_OPTIONS[@]}. 240 241 -e <example_binary_dir> 242 Path to directory that holds all example binaries 243 244 -g <gemm_shape_file> 245 Path to gemm shape csv file 246 247 -c <gemm_config_file> 248 Path to gemm config csv file 249 250 -d <data_type> 251 Data type option with which to run benchmark examples 252 Default: ${DEFAULT_DATA_TYPE} 253 Supported options: 254 Strategy : Data Types 255 Native : f32 256 Reshaped : f32, f16, qasymm8 257 Reshaped RHS Only : f32, f16, qasymm8 258 259 -o <out_dir> 260 Path to output directory that holds output json files 261 Default: ${DEFAULT_OUT_DIR} 262 263 -i <id_experiment_start> 264 ID of the first experiment. 265 Default: ${DEFAULT_ID_EXPERIMENT_START} 266 267 -n <num_experiments> 268 Total number of experiments to execute in this session. [1-all] 269 Default: ${DEFAULT_NUM_EXPERIMENTS} 270 271 -t <output_extension> 272 Output file extension. 273 Default: ${DEFAULT_OUT_EXTENSION} 274 275 -m <tuner_mode> 276 OpenCL tuner mode. 277 Default: ${DEFAULT_TUNER_MODE} 278 279EOF 280# Print help messages about gemm shapes and various gemm configs 281$HELP && help_gemm_shape_file 282$HELP && ( [ "${STRATEGY_OPTION}" == "" ] || [ "${STRATEGY_OPTION}" == "native" ] ) && help_gemm_config_file_native 283$HELP && ( [ "${STRATEGY_OPTION}" == "" ] || [ "${STRATEGY_OPTION}" == "reshaped_rhs_only" ] ) && help_gemm_config_file_reshaped_rhs_only 284$HELP && ( [ "${STRATEGY_OPTION}" == "" ] || [ "${STRATEGY_OPTION}" == "reshaped" ] ) && help_gemm_config_file_reshaped 285exit 1 286} 287 288####################################### 289# Print error message and exit with Error. 290# Globals: 291# None 292# Arguments: 293# $1 - Error message 294# Returns: 295# None 296####################################### 297function error_msg() { 298 echo "Error: $1" 1>&2 299 exit 1 300} 301 302####################################### 303# Convert string to lower-case 304# Globals: 305# None 306# Arguments: 307# target - String 308# Returns: 309# (stdout) - String in lowercase 310####################################### 311function to_lower() { 312 local target=$1 313 echo "$target" | tr '[:upper:]' '[:lower:]' 314} 315 316####################################### 317# Test if the argument is an integer 318# Globals: 319# None 320# Arguments: 321# in - Input 322# Returns: 323# true/false 324####################################### 325function is_integer() { 326 local in=$1 327 [ "$in" -eq "$in" ] 2> /dev/null 328} 329 330####################################### 331# Test if a string is in an array of strings 332# Globals: 333# None 334# Arguments: 335# target - String to test 336# array - Array of strings to search 337# Returns: 338# true/false 339####################################### 340function arr_contains() { 341 local target=$1 342 shift 343 local array 344 array=("$@") 345 for s in "${array[@]}" 346 do 347 [ "$s" == "${target}" ] && return 348 done 349 false 350} 351 352####################################### 353# Run a single example with all tunable gemm configurations on all gemm parameters 354# Globals: 355# OUT_DIR 356# OUT_EXTENSION 357# TUNER_MODE 358# EXAMPLE_BIN_DIR 359# NUM_ITERATION 360# GEMM_CONFIGS_FILE 361# GEMM_SHAPES_FILE 362# STRATEGY_OPTION 363# DATA_TYPE 364# OUT_DIR 365# ID_EXPERIMENT_START 366# NUM_EXPERIMENTS 367 368# Arguments: 369# example_bin Name of the example binary to run 370# Returns: 371# None 372####################################### 373function run() { 374 local example_bin=$1 375 echo "Running experiments for ${example_bin}" 1>&2 376 local example_args 377 local json_filename 378 local expr_count=0 379 # Total number of experiments available 380 local num_experiments_total 381 # Total number of experiment runs scheduled for this session 382 local num_experiments_session 383 local id_experiment_start 384 local id_experiment_end 385 local array_shapes 386 local array_configs 387 local array_shapes_len 388 local array_configs_len 389 local array_shapes_idx 390 local array_configs_idx 391 local match_expression_shape="^([^,]*,){3}[^,]*$" 392 local match_expression_config="^(\s*[0-9]+\s*,)+\s*[0-9]\s*$" 393 local shapes_list_cmd="grep -E "$match_expression_shape" "${GEMM_SHAPES_FILE}"" 394 local configs_list_cmd="grep -E "$match_expression_config" "${GEMM_CONFIGS_FILE}"" 395 396 # Create array from CSV file 397 array_shapes=($( $shapes_list_cmd )) 398 array_configs=($( $configs_list_cmd )) 399 400 # Get array length 401 array_shapes_len=${#array_shapes[@]} 402 array_configs_len=${#array_configs[@]} 403 404 # Get the total number of experiments available 405 (( num_experiments_total=${array_shapes_len} * ${array_configs_len} )) 406 407 # Get the number of experiments to execute in this session 408 if [ ${NUM_EXPERIMENTS} == ${DEFAULT_NUM_EXPERIMENTS} ] 409 then 410 (( num_experiments_session=${array_shapes_len} * ${array_configs_len} )) 411 else 412 num_experiments_session=$NUM_EXPERIMENTS 413 fi 414 415 # Id experiment start 416 id_experiment_start=${ID_EXPERIMENT_START} 417 418 # Id experiment end 419 (( id_experiment_end=(${num_experiments_session} + ${id_experiment_start} - 1) )) 420 421 # Check if the id experiment end is grater than or equal to the total number of experiments available. 422 # If the condition is satisfied, clamp the id experiment end 423 if [ "$id_experiment_end" -ge "$num_experiments_total" ] 424 then 425 echo "Clamping idx experiment end" 1>&2 426 (( id_experiment_end=${num_experiments_total} - 1 )) 427 (( num_experiments_session=${id_experiment_start} + ${id_experiment_end} + 1 )) 428 fi 429 430 # Time elapsed since the beginning in seconds 431 local time_elapsed_s 432 # Time estimated to finish in seconds 433 local time_est_s 434 echo "Running a total number of ${num_experiments_session} experiments" 1>&2 435 echo "Experiment idx start/end [${id_experiment_start}, ${id_experiment_end}]" 1>&2 436 437 # Run experiments 438 for i in $(seq $id_experiment_start $id_experiment_end); 439 do 440 (( array_shapes_idx=${i} / ${array_configs_len} )) 441 (( array_configs_idx=${i} % ${array_configs_len} )) 442 443 gemm_shape=${array_shapes[$array_shapes_idx]} 444 gemm_config=${array_configs[$array_configs_idx]} 445 446 echo "Running shape[$array_shapes_idx]=$gemm_shape with config[$array_configs_idx]=$gemm_config" 1>&2 447 448 example_args="${gemm_shape},${gemm_config},--type=${DATA_TYPE},--tuner-mode=${TUNER_MODE}" 449 json_filename="${STRATEGY_OPTION}_${gemm_shape}_${gemm_config}_${DATA_TYPE}" 450 # Replace "," with "_" 451 json_filename=${json_filename//,/_} 452 453 # Run experiment 454 ${EXAMPLE_BIN_DIR}/${example_bin} --example_args=${example_args} --iterations=${NUM_ITERATION} --json-file=${OUT_DIR}/${json_filename}.${OUT_EXTENSION} --instruments=OPENCL_TIMER_MS 455 # Print progress 456 (( expr_count++ )) 457 print_progress ${expr_count} ${num_experiments_session} 458 # Print time statistics 459 time_elapsed_s=$SECONDS 460 echo "Time elapsed since beginning: $(( $time_elapsed_s / 60 ))m $(( $time_elapsed_s % 60 ))s" 1>&2 461 (( time_est_s=(${num_experiments_session} - ${expr_count}) * ${time_elapsed_s} / ${expr_count} )) 462 echo "Time estimated to finish: $(( $time_est_s / 60 ))m $(( $time_est_s % 60 ))s" 1>&2 463 echo "Done." 1>&2 464 done 465 466 echo "Finished running all configs for ${example_bin}" 1>&2 467 echo "All results saved to ${OUT_DIR}" 1>&2 468} 469 470####################################### 471# Print the progress of the current session 472# Globals: 473# None 474# Arguments: 475# current Current number of items 476# total Total number of items 477# Returns: 478# None 479####################################### 480function print_progress() { 481 local current 482 local total 483 current=$1 484 total=$2 485 # Width of progress bar 486 local width 487 width=20 488 (( current_width= $width * current / total )) 489 echo -n -e "Progress [" 1>&2 490 for i in $(seq 1 ${width}); do 491 if [[ $i -le ${current_width} ]]; then 492 echo -n "#" 1>&2 493 else 494 echo -n " " 1>&2 495 fi 496 done 497 echo "] $current / $total Experiments" 1>&2 498} 499 500# Functions }}} 501 502# Main: Main script {{{ 503# Path to directory containing all benchmark examples binaries 504EXAMPLE_BIN_DIR="" 505# Path to gemm shapes file 506GEMM_SHAPES_FILE="" 507# Path to gemm configs file 508GEMM_CONFIGS_FILE="" 509# Strategy option 510STRATEGY_OPTION="" 511# Data type to use 512DATA_TYPE=${DEFAULT_DATA_TYPE} 513# Path to output directory 514OUT_DIR=${DEFAULT_OUT_DIR} 515# ID of the first experiment 516ID_EXPERIMENT_START=${DEFAULT_ID_EXPERIMENT_START} 517# Total number of experiments to execute in this session 518NUM_EXPERIMENTS=${DEFAULT_NUM_EXPERIMENTS} 519# Output benchmark result file extension 520OUT_EXTENSION=${DEFAULT_OUT_EXTENSION} 521# OpenCL tuner mode 522TUNER_MODE=${DEFAULT_TUNER_MODE} 523# Toggle help 524HELP=false 525 526# Obtain options 527while getopts "hs:e:g:c:d:o:i:n:t:m:" opt; do 528 case "$opt" in 529 h) HELP=true ;; 530 s) STRATEGY_OPTION=$(to_lower "${OPTARG}");; 531 e) EXAMPLE_BIN_DIR="${OPTARG}";; 532 g) GEMM_SHAPES_FILE="${OPTARG}";; 533 c) GEMM_CONFIGS_FILE="${OPTARG}";; 534 d) DATA_TYPE=$(to_lower "${OPTARG}");; 535 o) OUT_DIR="${OPTARG}";; 536 i) ID_EXPERIMENT_START="${OPTARG}";; 537 n) NUM_EXPERIMENTS="${OPTARG}";; 538 t) OUT_EXTENSION="${OPTARG}";; 539 m) TUNER_MODE="${OPTARG}";; 540 esac 541done 542shift $((OPTIND - 1)) 543 544# Lazily print usage (after arguments have been parsed) 545$HELP && 546 usage 547 548# Parse and validate options 549# Verify all compulsory arguments are passed in 550( [ ! -z "${STRATEGY_OPTION}" ] && [ ! -z "${EXAMPLE_BIN_DIR}" ] && [ ! -z "${GEMM_SHAPES_FILE}" ] && [ ! -z "${GEMM_CONFIGS_FILE}" ] ) || 551 usage 552 553# Verify example binaries directory exists 554[ -d "${EXAMPLE_BIN_DIR}" ] || 555 error_msg "${EXAMPLE_BIN_DIR} does not exist." 556 557# Verify all benchmark example binaries exist 558[ -f "${EXAMPLE_BIN_DIR}/${EXAMPLE_BIN_RESHAPED_RHS_ONLY}" ] || 559 error_msg "Cannot find ${EXAMPLE_BIN_RESHAPED_RHS_ONLY} at ${EXAMPLE_BIN_DIR}" 560 561# Verify Gemm shapes file exists 562[ -f "${GEMM_SHAPES_FILE}" ] || 563 error_msg "Cannot find gemm shapes file ${GEMM_SHAPES_FILE}" 564 565# Verify Gemm configs file exists 566[ -f "${GEMM_CONFIGS_FILE}" ] || 567 error_msg "Cannot find gemm configs file ${GEMM_CONFIGS_FILE}" 568 569# Verify strategy option is valid 570arr_contains "${STRATEGY_OPTION}" "${ALL_STRATEGY_OPTIONS[@]}" || 571 error_msg "Does not support strategy ${STRATEGY_OPTION}" 572 573# Verify data type option is valid 574arr_contains "${DATA_TYPE}" "${ALL_DATA_TYPE_OPTIONS[@]}" || 575 error_msg "Does not support data type ${DATA_TYPE}" 576 577# Make sure existing benchmark outputs are not overwritten 578[ ! -d "${OUT_DIR}" ] || 579 error_msg "Output directory ${OUT_DIR} already exists!" 580 581# Make output directory 582echo "Making output directory ${OUT_DIR}" 1>&2 583mkdir -p ${OUT_DIR} || error_msg "Failed to make output directory ${OUT_DIR}" 584 585# Run selected strategy with all configurations 586# Restart the built-in timer 587SECONDS=0 588if [ "$DATA_TYPE" == "qasymm8" ]; then 589 [ "${STRATEGY_OPTION}" == "reshaped_rhs_only" ] && run $EXAMPLE_BIN_RESHAPED_RHS_ONLY_LOWP 590 [ "${STRATEGY_OPTION}" == "reshaped" ] && run $EXAMPLE_BIN_RESHAPED_LOWP 591else 592 [ "${STRATEGY_OPTION}" == "native" ] && run $EXAMPLE_BIN_NATIVE 593 [ "${STRATEGY_OPTION}" == "reshaped_rhs_only" ] && run $EXAMPLE_BIN_RESHAPED_RHS_ONLY 594 [ "${STRATEGY_OPTION}" == "reshaped" ] && run $EXAMPLE_BIN_RESHAPED 595fi 596# Main: Main script }}} 597