41 static void GEMMBenchmark(benchmark::State& state,  in GEMMBenchmark()  argument
48     state.SkipWithError("cpuinfo initialization failed");  in GEMMBenchmark()
51   if (isa_check && !isa_check(state)) {  in GEMMBenchmark()
55   const size_t mc = state.range(0);  in GEMMBenchmark()
56   const size_t nc = state.range(1);  in GEMMBenchmark()
57   const size_t kc = state.range(2);  in GEMMBenchmark()
91   for (auto _ : state) {  in GEMMBenchmark()
92     // Use circular buffers (exceeding cache size) and prefetch to control cache state:  in GEMMBenchmark()
96     state.PauseTiming();  in GEMMBenchmark()
99     state.ResumeTiming();  in GEMMBenchmark()
117     state.counters["cpufreq"] = cpu_frequency;  in GEMMBenchmark()
120   state.counters["OPS"] = benchmark::Counter(  in GEMMBenchmark()
121     uint64_t(state.iterations()) * 2 * mc * nc * kc, benchmark::Counter::kIsRate);  in GEMMBenchmark()
158 static void GemmlowpBenchmark(benchmark::State& state, uint32_t threads)  in GemmlowpBenchmark()  argument
160   const size_t mc = state.range(0);  in GemmlowpBenchmark()
161   const size_t nc = state.range(1);  in GemmlowpBenchmark()
162   const size_t kc = state.range(2);  in GemmlowpBenchmark()
190   for (auto _ : state) {  in GemmlowpBenchmark()
191     state.PauseTiming();  in GemmlowpBenchmark()
194     state.ResumeTiming();  in GemmlowpBenchmark()
206     state.counters["cpufreq"] = cpu_frequency;  in GemmlowpBenchmark()
209   state.counters["OPS"] = benchmark::Counter(  in GemmlowpBenchmark()
210     uint64_t(state.iterations()) * 2 * mc * nc * kc, benchmark::Counter::kIsRate);  in GemmlowpBenchmark()
213 static void gemmlowp_st(benchmark::State& state, const char* net)  in gemmlowp_st()  argument
215   GemmlowpBenchmark(state, 1);  in gemmlowp_st()
221 static void RuyBenchmark(benchmark::State& state, size_t threads)  in RuyBenchmark()  argument
223   const size_t mc = state.range(0);  in RuyBenchmark()
224   const size_t nc = state.range(1);  in RuyBenchmark()
225   const size_t kc = state.range(2);  in RuyBenchmark()
280   for (auto _ : state) {  in RuyBenchmark()
281     // Use circular buffers (exceeding cache size) and prefetch to control cache state:  in RuyBenchmark()
286     state.PauseTiming();  in RuyBenchmark()
289     state.ResumeTiming();  in RuyBenchmark()
300     state.counters["cpufreq"] = cpu_frequency;  in RuyBenchmark()
303   state.counters["OPS"] = benchmark::Counter(  in RuyBenchmark()
304     uint64_t(state.iterations()) * 2 * mc * nc * kc, benchmark::Counter::kIsRate);  in RuyBenchmark()
307 static void ruy_st(benchmark::State& state, const char* net)  in ruy_st()  argument
309   RuyBenchmark(state, 1);  in ruy_st()
315 …static void qu8_gemm_4x8__aarch32_neon_mlal_lane_cortex_a53(benchmark::State& state, const char* n…  in qu8_gemm_4x8__aarch32_neon_mlal_lane_cortex_a53()  argument
316     GEMMBenchmark(state,  in qu8_gemm_4x8__aarch32_neon_mlal_lane_cortex_a53()
321 …static void qu8_gemm_4x8__aarch32_neon_mlal_lane_prfm_cortex_a53(benchmark::State& state, const ch…  in qu8_gemm_4x8__aarch32_neon_mlal_lane_prfm_cortex_a53()  argument
322 …GEMMBenchmark(state, xnn_qu8_gemm_minmax_rndnu_ukernel_4x8__aarch32_neon_mlal_lane_prfm_cortex_a53,  in qu8_gemm_4x8__aarch32_neon_mlal_lane_prfm_cortex_a53()
326 …static void qu8_gemm_4x8__aarch32_neon_mlal_lane_cortex_a7(benchmark::State& state, const char* ne…  in qu8_gemm_4x8__aarch32_neon_mlal_lane_cortex_a7()  argument
327     GEMMBenchmark(state,  in qu8_gemm_4x8__aarch32_neon_mlal_lane_cortex_a7()
332 …static void qu8_gemm_4x8__aarch32_neon_mlal_lane_prfm_cortex_a7(benchmark::State& state, const cha…  in qu8_gemm_4x8__aarch32_neon_mlal_lane_prfm_cortex_a7()  argument
333 … GEMMBenchmark(state, xnn_qu8_gemm_minmax_rndnu_ukernel_4x8__aarch32_neon_mlal_lane_prfm_cortex_a7,  in qu8_gemm_4x8__aarch32_neon_mlal_lane_prfm_cortex_a7()
337   static void qu8_gemm_4x8__aarch32_neon_mlal_lane_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_4x8__aarch32_neon_mlal_lane_ld64()  argument
338     GEMMBenchmark(state,  in qu8_gemm_4x8__aarch32_neon_mlal_lane_ld64()
343 …static void qu8_gemm_4x8__aarch32_neon_mlal_lane_prfm_ld64(benchmark::State& state, const char* ne…  in qu8_gemm_4x8__aarch32_neon_mlal_lane_prfm_ld64()  argument
344     GEMMBenchmark(state, xnn_qu8_gemm_minmax_rndnu_ukernel_4x8__aarch32_neon_mlal_lane_prfm_ld64,  in qu8_gemm_4x8__aarch32_neon_mlal_lane_prfm_ld64()
348 …static void qu8_gemm_1x8__aarch32_neon_mlal_lane_cortex_a7(benchmark::State& state, const char* ne…  in qu8_gemm_1x8__aarch32_neon_mlal_lane_cortex_a7()  argument
349     GEMMBenchmark(state,  in qu8_gemm_1x8__aarch32_neon_mlal_lane_cortex_a7()
354 …static void qu8_gemm_1x8__aarch32_neon_mlal_lane_prfm_cortex_a7(benchmark::State& state, const cha…  in qu8_gemm_1x8__aarch32_neon_mlal_lane_prfm_cortex_a7()  argument
355 … GEMMBenchmark(state, xnn_qu8_gemm_minmax_rndnu_ukernel_1x8__aarch32_neon_mlal_lane_prfm_cortex_a7,  in qu8_gemm_1x8__aarch32_neon_mlal_lane_prfm_cortex_a7()
371 …static void qu8_gemm_4x16c4__aarch64_neondot_cortex_a55(benchmark::State& state, const char* net) {  in BENCHMARK_GEMM()
372     GEMMBenchmark(state,  in BENCHMARK_GEMM()
377   static void qu8_gemm_4x16c4__aarch64_neondot_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_4x16c4__aarch64_neondot_ld128()  argument
378     GEMMBenchmark(state,  in qu8_gemm_4x16c4__aarch64_neondot_ld128()
383   static void qu8_gemm_4x8c4__aarch64_neondot_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_4x8c4__aarch64_neondot_ld128()  argument
384     GEMMBenchmark(state,  in qu8_gemm_4x8c4__aarch64_neondot_ld128()
390   static void qu8_gemm_4x8c4__aarch64_neondot_cortex_a55(benchmark::State& state, const char* net) {  in qu8_gemm_4x8c4__aarch64_neondot_cortex_a55()  argument
391     GEMMBenchmark(state,  in qu8_gemm_4x8c4__aarch64_neondot_cortex_a55()
396 …static void qu8_gemm_4x16__aarch64_neon_mlal_lane_cortex_a53(benchmark::State& state, const char* …  in qu8_gemm_4x16__aarch64_neon_mlal_lane_cortex_a53()  argument
397     GEMMBenchmark(state,  in qu8_gemm_4x16__aarch64_neon_mlal_lane_cortex_a53()
403 …static void qu8_gemm_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53(benchmark::State& state, const c…  in qu8_gemm_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53()  argument
404     GEMMBenchmark(state,  in qu8_gemm_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53()
410   static void qu8_gemm_4x16__aarch64_neon_mlal_lane_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_4x16__aarch64_neon_mlal_lane_ld64()  argument
411     GEMMBenchmark(state,  in qu8_gemm_4x16__aarch64_neon_mlal_lane_ld64()
417 …static void qu8_gemm_4x16__aarch64_neon_mlal_lane_prfm_ld64(benchmark::State& state, const char* n…  in qu8_gemm_4x16__aarch64_neon_mlal_lane_prfm_ld64()  argument
418     GEMMBenchmark(state,  in qu8_gemm_4x16__aarch64_neon_mlal_lane_prfm_ld64()
424 …static void qu8_gemm_4x16__aarch64_neon_mlal_lane_cortex_a75(benchmark::State& state, const char* …  in qu8_gemm_4x16__aarch64_neon_mlal_lane_cortex_a75()  argument
425     GEMMBenchmark(state,  in qu8_gemm_4x16__aarch64_neon_mlal_lane_cortex_a75()
431 …static void qu8_gemm_4x16__aarch64_neon_mlal_lane_prfm_cortex_a75(benchmark::State& state, const c…  in qu8_gemm_4x16__aarch64_neon_mlal_lane_prfm_cortex_a75()  argument
432     GEMMBenchmark(state,  in qu8_gemm_4x16__aarch64_neon_mlal_lane_prfm_cortex_a75()
452   static void qu8_gemm_1x8c4__neondot(benchmark::State& state, const char* net) {  in BENCHMARK_GEMM()
453     GEMMBenchmark(state,  in BENCHMARK_GEMM()
458   static void qu8_gemm_2x8c4__neondot(benchmark::State& state, const char* net) {  in qu8_gemm_2x8c4__neondot()  argument
459     GEMMBenchmark(state,  in qu8_gemm_2x8c4__neondot()
464   static void qu8_gemm_3x8c4__neondot(benchmark::State& state, const char* net) {  in qu8_gemm_3x8c4__neondot()  argument
465     GEMMBenchmark(state,  in qu8_gemm_3x8c4__neondot()
470   static void qu8_gemm_4x8c4__neondot(benchmark::State& state, const char* net) {  in qu8_gemm_4x8c4__neondot()  argument
471     GEMMBenchmark(state,  in qu8_gemm_4x8c4__neondot()
476   static void qu8_gemm_5x8c4__neondot(benchmark::State& state, const char* net) {  in qu8_gemm_5x8c4__neondot()  argument
477     GEMMBenchmark(state,  in qu8_gemm_5x8c4__neondot()
482   static void qu8_gemm_6x8c4__neondot(benchmark::State& state, const char* net) {  in qu8_gemm_6x8c4__neondot()  argument
483     GEMMBenchmark(state,  in qu8_gemm_6x8c4__neondot()
488   static void qu8_gemm_8x8c4__neondot(benchmark::State& state, const char* net) {  in qu8_gemm_8x8c4__neondot()  argument
489     GEMMBenchmark(state,  in qu8_gemm_8x8c4__neondot()
494   static void qu8_gemm_1x16c4__neondot(benchmark::State& state, const char* net) {  in qu8_gemm_1x16c4__neondot()  argument
495     GEMMBenchmark(state,  in qu8_gemm_1x16c4__neondot()
500   static void qu8_gemm_2x16c4__neondot(benchmark::State& state, const char* net) {  in qu8_gemm_2x16c4__neondot()  argument
501     GEMMBenchmark(state,  in qu8_gemm_2x16c4__neondot()
506   static void qu8_gemm_3x16c4__neondot(benchmark::State& state, const char* net) {  in qu8_gemm_3x16c4__neondot()  argument
507     GEMMBenchmark(state,  in qu8_gemm_3x16c4__neondot()
512   static void qu8_gemm_4x16c4__neondot(benchmark::State& state, const char* net) {  in qu8_gemm_4x16c4__neondot()  argument
513     GEMMBenchmark(state,  in qu8_gemm_4x16c4__neondot()
518   static void qu8_gemm_5x16c4__neondot(benchmark::State& state, const char* net) {  in qu8_gemm_5x16c4__neondot()  argument
519     GEMMBenchmark(state,  in qu8_gemm_5x16c4__neondot()
524   static void qu8_gemm_6x16c4__neondot(benchmark::State& state, const char* net) {  in qu8_gemm_6x16c4__neondot()  argument
525     GEMMBenchmark(state,  in qu8_gemm_6x16c4__neondot()
530   static void qu8_gemm_8x16c4__neondot(benchmark::State& state, const char* net) {  in qu8_gemm_8x16c4__neondot()  argument
531     GEMMBenchmark(state,  in qu8_gemm_8x16c4__neondot()
536   static void qu8_gemm_1x32c4__neondot(benchmark::State& state, const char* net) {  in qu8_gemm_1x32c4__neondot()  argument
537     GEMMBenchmark(state,  in qu8_gemm_1x32c4__neondot()
542   static void qu8_gemm_2x32c4__neondot(benchmark::State& state, const char* net) {  in qu8_gemm_2x32c4__neondot()  argument
543     GEMMBenchmark(state,  in qu8_gemm_2x32c4__neondot()
548   static void qu8_gemm_3x32c4__neondot(benchmark::State& state, const char* net) {  in qu8_gemm_3x32c4__neondot()  argument
549     GEMMBenchmark(state,  in qu8_gemm_3x32c4__neondot()
576   static void qu8_gemm_1x8__neon_mlal_lane(benchmark::State& state, const char* net) {  in BENCHMARK_GEMM()
577     GEMMBenchmark(state,  in BENCHMARK_GEMM()
582   static void qu8_gemm_2x8__neon_mlal_lane(benchmark::State& state, const char* net) {  in qu8_gemm_2x8__neon_mlal_lane()  argument
583     GEMMBenchmark(state,  in qu8_gemm_2x8__neon_mlal_lane()
588   static void qu8_gemm_3x8__neon_mlal_lane(benchmark::State& state, const char* net) {  in qu8_gemm_3x8__neon_mlal_lane()  argument
589     GEMMBenchmark(state,  in qu8_gemm_3x8__neon_mlal_lane()
594   static void qu8_gemm_4x8__neon_mlal_lane(benchmark::State& state, const char* net) {  in qu8_gemm_4x8__neon_mlal_lane()  argument
595     GEMMBenchmark(state,  in qu8_gemm_4x8__neon_mlal_lane()
600   static void qu8_gemm_6x8__neon_mlal_lane(benchmark::State& state, const char* net) {  in qu8_gemm_6x8__neon_mlal_lane()  argument
601     GEMMBenchmark(state,  in qu8_gemm_6x8__neon_mlal_lane()
606   static void qu8_gemm_1x16__neon_mlal_lane(benchmark::State& state, const char* net) {  in qu8_gemm_1x16__neon_mlal_lane()  argument
607     GEMMBenchmark(state,  in qu8_gemm_1x16__neon_mlal_lane()
612   static void qu8_gemm_2x16__neon_mlal_lane(benchmark::State& state, const char* net) {  in qu8_gemm_2x16__neon_mlal_lane()  argument
613     GEMMBenchmark(state,  in qu8_gemm_2x16__neon_mlal_lane()
618   static void qu8_gemm_3x16__neon_mlal_lane(benchmark::State& state, const char* net) {  in qu8_gemm_3x16__neon_mlal_lane()  argument
619     GEMMBenchmark(state,  in qu8_gemm_3x16__neon_mlal_lane()
624   static void qu8_gemm_4x16__neon_mlal_lane(benchmark::State& state, const char* net) {  in qu8_gemm_4x16__neon_mlal_lane()  argument
625     GEMMBenchmark(state,  in qu8_gemm_4x16__neon_mlal_lane()
630   static void qu8_gemm_6x16__neon_mlal_lane(benchmark::State& state, const char* net) {  in qu8_gemm_6x16__neon_mlal_lane()  argument
631     GEMMBenchmark(state,  in qu8_gemm_6x16__neon_mlal_lane()
651   static void qu8_gemm_1x1c4__armsimd32(benchmark::State& state, const char* net) {  in BENCHMARK_GEMM()
652     GEMMBenchmark(state,  in BENCHMARK_GEMM()
657   static void qu8_gemm_2x1c4__armsimd32(benchmark::State& state, const char* net) {  in qu8_gemm_2x1c4__armsimd32()  argument
658     GEMMBenchmark(state,  in qu8_gemm_2x1c4__armsimd32()
663   static void qu8_gemm_1x2c4__armsimd32(benchmark::State& state, const char* net) {  in qu8_gemm_1x2c4__armsimd32()  argument
664     GEMMBenchmark(state,  in qu8_gemm_1x2c4__armsimd32()
669   static void qu8_gemm_2x2c4__armsimd32(benchmark::State& state, const char* net) {  in qu8_gemm_2x2c4__armsimd32()  argument
670     GEMMBenchmark(state,  in qu8_gemm_2x2c4__armsimd32()
684   static void qu8_gemm_1x16c8__avx512skx(benchmark::State& state, const char* net) {  in BENCHMARK_GEMM()
685     GEMMBenchmark(state,  in BENCHMARK_GEMM()
691   static void qu8_gemm_2x16c8__avx512skx(benchmark::State& state, const char* net) {  in qu8_gemm_2x16c8__avx512skx()  argument
692     GEMMBenchmark(state,  in qu8_gemm_2x16c8__avx512skx()
698   static void qu8_gemm_3x16c8__avx512skx(benchmark::State& state, const char* net) {  in qu8_gemm_3x16c8__avx512skx()  argument
699     GEMMBenchmark(state,  in qu8_gemm_3x16c8__avx512skx()
705   static void qu8_gemm_4x16c8__avx512skx(benchmark::State& state, const char* net) {  in qu8_gemm_4x16c8__avx512skx()  argument
706     GEMMBenchmark(state,  in qu8_gemm_4x16c8__avx512skx()
712   static void qu8_gemm_1x8c8__avx2(benchmark::State& state, const char* net) {  in qu8_gemm_1x8c8__avx2()  argument
713     GEMMBenchmark(state,  in qu8_gemm_1x8c8__avx2()
719   static void qu8_gemm_2x8c8__avx2(benchmark::State& state, const char* net) {  in qu8_gemm_2x8c8__avx2()  argument
720     GEMMBenchmark(state,  in qu8_gemm_2x8c8__avx2()
726   static void qu8_gemm_3x8c8__avx2(benchmark::State& state, const char* net) {  in qu8_gemm_3x8c8__avx2()  argument
727     GEMMBenchmark(state,  in qu8_gemm_3x8c8__avx2()
733   static void qu8_gemm_1x4c2__xop_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_1x4c2__xop_ld64()  argument
734     GEMMBenchmark(state,  in qu8_gemm_1x4c2__xop_ld64()
740   static void qu8_gemm_2x4c2__xop_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_2x4c2__xop_ld64()  argument
741     GEMMBenchmark(state,  in qu8_gemm_2x4c2__xop_ld64()
747   static void qu8_gemm_3x4c2__xop_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_3x4c2__xop_ld64()  argument
748     GEMMBenchmark(state,  in qu8_gemm_3x4c2__xop_ld64()
754   static void qu8_gemm_4x4c2__xop_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_4x4c2__xop_ld64()  argument
755     GEMMBenchmark(state,  in qu8_gemm_4x4c2__xop_ld64()
761   static void qu8_gemm_1x4c2__xop_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_1x4c2__xop_ld128()  argument
762     GEMMBenchmark(state,  in qu8_gemm_1x4c2__xop_ld128()
768   static void qu8_gemm_2x4c2__xop_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_2x4c2__xop_ld128()  argument
769     GEMMBenchmark(state,  in qu8_gemm_2x4c2__xop_ld128()
775   static void qu8_gemm_3x4c2__xop_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_3x4c2__xop_ld128()  argument
776     GEMMBenchmark(state,  in qu8_gemm_3x4c2__xop_ld128()
782   static void qu8_gemm_4x4c2__xop_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_4x4c2__xop_ld128()  argument
783     GEMMBenchmark(state,  in qu8_gemm_4x4c2__xop_ld128()
789   static void qu8_gemm_1x4c8__xop_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_1x4c8__xop_ld64()  argument
790     GEMMBenchmark(state,  in qu8_gemm_1x4c8__xop_ld64()
796   static void qu8_gemm_2x4c8__xop_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_2x4c8__xop_ld64()  argument
797     GEMMBenchmark(state,  in qu8_gemm_2x4c8__xop_ld64()
803   static void qu8_gemm_3x4c8__xop_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_3x4c8__xop_ld64()  argument
804     GEMMBenchmark(state,  in qu8_gemm_3x4c8__xop_ld64()
810   static void qu8_gemm_1x4c8__xop_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_1x4c8__xop_ld128()  argument
811     GEMMBenchmark(state,  in qu8_gemm_1x4c8__xop_ld128()
817   static void qu8_gemm_2x4c8__xop_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_2x4c8__xop_ld128()  argument
818     GEMMBenchmark(state,  in qu8_gemm_2x4c8__xop_ld128()
824   static void qu8_gemm_3x4c8__xop_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_3x4c8__xop_ld128()  argument
825     GEMMBenchmark(state,  in qu8_gemm_3x4c8__xop_ld128()
831   static void qu8_gemm_1x4c2__avx_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_1x4c2__avx_ld64()  argument
832     GEMMBenchmark(state,  in qu8_gemm_1x4c2__avx_ld64()
838   static void qu8_gemm_2x4c2__avx_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_2x4c2__avx_ld64()  argument
839     GEMMBenchmark(state,  in qu8_gemm_2x4c2__avx_ld64()
845   static void qu8_gemm_3x4c2__avx_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_3x4c2__avx_ld64()  argument
846     GEMMBenchmark(state,  in qu8_gemm_3x4c2__avx_ld64()
852   static void qu8_gemm_4x4c2__avx_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_4x4c2__avx_ld64()  argument
853     GEMMBenchmark(state,  in qu8_gemm_4x4c2__avx_ld64()
859   static void qu8_gemm_1x4c2__avx_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_1x4c2__avx_ld128()  argument
860     GEMMBenchmark(state,  in qu8_gemm_1x4c2__avx_ld128()
866   static void qu8_gemm_2x4c2__avx_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_2x4c2__avx_ld128()  argument
867     GEMMBenchmark(state,  in qu8_gemm_2x4c2__avx_ld128()
873   static void qu8_gemm_3x4c2__avx_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_3x4c2__avx_ld128()  argument
874     GEMMBenchmark(state,  in qu8_gemm_3x4c2__avx_ld128()
880   static void qu8_gemm_4x4c2__avx_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_4x4c2__avx_ld128()  argument
881     GEMMBenchmark(state,  in qu8_gemm_4x4c2__avx_ld128()
887   static void qu8_gemm_1x4c8__avx_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_1x4c8__avx_ld64()  argument
888     GEMMBenchmark(state,  in qu8_gemm_1x4c8__avx_ld64()
894   static void qu8_gemm_2x4c8__avx_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_2x4c8__avx_ld64()  argument
895     GEMMBenchmark(state,  in qu8_gemm_2x4c8__avx_ld64()
901   static void qu8_gemm_3x4c8__avx_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_3x4c8__avx_ld64()  argument
902     GEMMBenchmark(state,  in qu8_gemm_3x4c8__avx_ld64()
908   static void qu8_gemm_1x4c8__avx_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_1x4c8__avx_ld128()  argument
909     GEMMBenchmark(state,  in qu8_gemm_1x4c8__avx_ld128()
915   static void qu8_gemm_2x4c8__avx_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_2x4c8__avx_ld128()  argument
916     GEMMBenchmark(state,  in qu8_gemm_2x4c8__avx_ld128()
922   static void qu8_gemm_3x4c8__avx_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_3x4c8__avx_ld128()  argument
923     GEMMBenchmark(state,  in qu8_gemm_3x4c8__avx_ld128()
929   static void qu8_gemm_1x4c2__sse41_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_1x4c2__sse41_ld64()  argument
930     GEMMBenchmark(state,  in qu8_gemm_1x4c2__sse41_ld64()
936   static void qu8_gemm_2x4c2__sse41_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_2x4c2__sse41_ld64()  argument
937     GEMMBenchmark(state,  in qu8_gemm_2x4c2__sse41_ld64()
943   static void qu8_gemm_3x4c2__sse41_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_3x4c2__sse41_ld64()  argument
944     GEMMBenchmark(state,  in qu8_gemm_3x4c2__sse41_ld64()
950   static void qu8_gemm_4x4c2__sse41_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_4x4c2__sse41_ld64()  argument
951     GEMMBenchmark(state,  in qu8_gemm_4x4c2__sse41_ld64()
957   static void qu8_gemm_1x4c2__sse41_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_1x4c2__sse41_ld128()  argument
958     GEMMBenchmark(state,  in qu8_gemm_1x4c2__sse41_ld128()
964   static void qu8_gemm_2x4c2__sse41_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_2x4c2__sse41_ld128()  argument
965     GEMMBenchmark(state,  in qu8_gemm_2x4c2__sse41_ld128()
971   static void qu8_gemm_3x4c2__sse41_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_3x4c2__sse41_ld128()  argument
972     GEMMBenchmark(state,  in qu8_gemm_3x4c2__sse41_ld128()
978   static void qu8_gemm_4x4c2__sse41_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_4x4c2__sse41_ld128()  argument
979     GEMMBenchmark(state,  in qu8_gemm_4x4c2__sse41_ld128()
985   static void qu8_gemm_1x4c8__sse41_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_1x4c8__sse41_ld64()  argument
986     GEMMBenchmark(state,  in qu8_gemm_1x4c8__sse41_ld64()
992   static void qu8_gemm_2x4c8__sse41_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_2x4c8__sse41_ld64()  argument
993     GEMMBenchmark(state,  in qu8_gemm_2x4c8__sse41_ld64()
999   static void qu8_gemm_3x4c8__sse41_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_3x4c8__sse41_ld64()  argument
1000     GEMMBenchmark(state,  in qu8_gemm_3x4c8__sse41_ld64()
1006   static void qu8_gemm_1x4c8__sse41_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_1x4c8__sse41_ld128()  argument
1007     GEMMBenchmark(state,  in qu8_gemm_1x4c8__sse41_ld128()
1013   static void qu8_gemm_2x4c8__sse41_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_2x4c8__sse41_ld128()  argument
1014     GEMMBenchmark(state,  in qu8_gemm_2x4c8__sse41_ld128()
1020   static void qu8_gemm_3x4c8__sse41_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_3x4c8__sse41_ld128()  argument
1021     GEMMBenchmark(state,  in qu8_gemm_3x4c8__sse41_ld128()
1027   static void qu8_gemm_1x4c2__sse2_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_1x4c2__sse2_ld64()  argument
1028     GEMMBenchmark(state,  in qu8_gemm_1x4c2__sse2_ld64()
1033   static void qu8_gemm_2x4c2__sse2_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_2x4c2__sse2_ld64()  argument
1034     GEMMBenchmark(state,  in qu8_gemm_2x4c2__sse2_ld64()
1039   static void qu8_gemm_3x4c2__sse2_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_3x4c2__sse2_ld64()  argument
1040     GEMMBenchmark(state,  in qu8_gemm_3x4c2__sse2_ld64()
1045   static void qu8_gemm_4x4c2__sse2_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_4x4c2__sse2_ld64()  argument
1046     GEMMBenchmark(state,  in qu8_gemm_4x4c2__sse2_ld64()
1051   static void qu8_gemm_1x4c2__sse2_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_1x4c2__sse2_ld128()  argument
1052     GEMMBenchmark(state,  in qu8_gemm_1x4c2__sse2_ld128()
1057   static void qu8_gemm_2x4c2__sse2_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_2x4c2__sse2_ld128()  argument
1058     GEMMBenchmark(state,  in qu8_gemm_2x4c2__sse2_ld128()
1063   static void qu8_gemm_3x4c2__sse2_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_3x4c2__sse2_ld128()  argument
1064     GEMMBenchmark(state,  in qu8_gemm_3x4c2__sse2_ld128()
1069   static void qu8_gemm_4x4c2__sse2_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_4x4c2__sse2_ld128()  argument
1070     GEMMBenchmark(state,  in qu8_gemm_4x4c2__sse2_ld128()
1075   static void qu8_gemm_1x4c8__sse2_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_1x4c8__sse2_ld64()  argument
1076     GEMMBenchmark(state,  in qu8_gemm_1x4c8__sse2_ld64()
1081   static void qu8_gemm_2x4c8__sse2_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_2x4c8__sse2_ld64()  argument
1082     GEMMBenchmark(state,  in qu8_gemm_2x4c8__sse2_ld64()
1087   static void qu8_gemm_3x4c8__sse2_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_3x4c8__sse2_ld64()  argument
1088     GEMMBenchmark(state,  in qu8_gemm_3x4c8__sse2_ld64()
1093   static void qu8_gemm_1x4c8__sse2_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_1x4c8__sse2_ld128()  argument
1094     GEMMBenchmark(state,  in qu8_gemm_1x4c8__sse2_ld128()
1099   static void qu8_gemm_2x4c8__sse2_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_2x4c8__sse2_ld128()  argument
1100     GEMMBenchmark(state,  in qu8_gemm_2x4c8__sse2_ld128()
1105   static void qu8_gemm_3x4c8__sse2_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_3x4c8__sse2_ld128()  argument
1106     GEMMBenchmark(state,  in qu8_gemm_3x4c8__sse2_ld128()
1184   static void qu8_gemm_1x4c2__wasmsimd_dot16x2_ld64(benchmark::State& state, const char* net) {  in BENCHMARK_GEMM()
1185     GEMMBenchmark(state,  in BENCHMARK_GEMM()
1190   static void qu8_gemm_2x4c2__wasmsimd_dot16x2_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_2x4c2__wasmsimd_dot16x2_ld64()  argument
1191     GEMMBenchmark(state,  in qu8_gemm_2x4c2__wasmsimd_dot16x2_ld64()
1196   static void qu8_gemm_3x4c2__wasmsimd_dot16x2_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_3x4c2__wasmsimd_dot16x2_ld64()  argument
1197     GEMMBenchmark(state,  in qu8_gemm_3x4c2__wasmsimd_dot16x2_ld64()
1202   static void qu8_gemm_4x4c2__wasmsimd_dot16x2_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_4x4c2__wasmsimd_dot16x2_ld64()  argument
1203     GEMMBenchmark(state,  in qu8_gemm_4x4c2__wasmsimd_dot16x2_ld64()
1209   static void qu8_gemm_1x4c2__wasmsimd_dot16x2_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_1x4c2__wasmsimd_dot16x2_ld128()  argument
1210     GEMMBenchmark(state,  in qu8_gemm_1x4c2__wasmsimd_dot16x2_ld128()
1215   static void qu8_gemm_2x4c2__wasmsimd_dot16x2_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_2x4c2__wasmsimd_dot16x2_ld128()  argument
1216     GEMMBenchmark(state,  in qu8_gemm_2x4c2__wasmsimd_dot16x2_ld128()
1221   static void qu8_gemm_3x4c2__wasmsimd_dot16x2_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_3x4c2__wasmsimd_dot16x2_ld128()  argument
1222     GEMMBenchmark(state,  in qu8_gemm_3x4c2__wasmsimd_dot16x2_ld128()
1227   static void qu8_gemm_4x4c2__wasmsimd_dot16x2_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_4x4c2__wasmsimd_dot16x2_ld128()  argument
1228     GEMMBenchmark(state,  in qu8_gemm_4x4c2__wasmsimd_dot16x2_ld128()
1234   static void qu8_gemm_1x4c2s4__wasmsimd_dot16x2_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_1x4c2s4__wasmsimd_dot16x2_ld64()  argument
1235     GEMMBenchmark(state,  in qu8_gemm_1x4c2s4__wasmsimd_dot16x2_ld64()
1240   static void qu8_gemm_2x4c2s4__wasmsimd_dot16x2_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_2x4c2s4__wasmsimd_dot16x2_ld64()  argument
1241     GEMMBenchmark(state,  in qu8_gemm_2x4c2s4__wasmsimd_dot16x2_ld64()
1246   static void qu8_gemm_3x4c2s4__wasmsimd_dot16x2_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_3x4c2s4__wasmsimd_dot16x2_ld64()  argument
1247     GEMMBenchmark(state,  in qu8_gemm_3x4c2s4__wasmsimd_dot16x2_ld64()
1252   static void qu8_gemm_4x4c2s4__wasmsimd_dot16x2_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_4x4c2s4__wasmsimd_dot16x2_ld64()  argument
1253     GEMMBenchmark(state,  in qu8_gemm_4x4c2s4__wasmsimd_dot16x2_ld64()
1259   static void qu8_gemm_1x4c2s4__wasmsimd_dot16x2_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_1x4c2s4__wasmsimd_dot16x2_ld128()  argument
1260     GEMMBenchmark(state,  in qu8_gemm_1x4c2s4__wasmsimd_dot16x2_ld128()
1265   static void qu8_gemm_2x4c2s4__wasmsimd_dot16x2_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_2x4c2s4__wasmsimd_dot16x2_ld128()  argument
1266     GEMMBenchmark(state,  in qu8_gemm_2x4c2s4__wasmsimd_dot16x2_ld128()
1271   static void qu8_gemm_3x4c2s4__wasmsimd_dot16x2_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_3x4c2s4__wasmsimd_dot16x2_ld128()  argument
1272     GEMMBenchmark(state,  in qu8_gemm_3x4c2s4__wasmsimd_dot16x2_ld128()
1277   static void qu8_gemm_4x4c2s4__wasmsimd_dot16x2_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_4x4c2s4__wasmsimd_dot16x2_ld128()  argument
1278     GEMMBenchmark(state,  in qu8_gemm_4x4c2s4__wasmsimd_dot16x2_ld128()
1284   static void qu8_gemm_1x4c8__wasmsimd_dot16x2_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_1x4c8__wasmsimd_dot16x2_ld64()  argument
1285     GEMMBenchmark(state,  in qu8_gemm_1x4c8__wasmsimd_dot16x2_ld64()
1290   static void qu8_gemm_2x4c8__wasmsimd_dot16x2_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_2x4c8__wasmsimd_dot16x2_ld64()  argument
1291     GEMMBenchmark(state,  in qu8_gemm_2x4c8__wasmsimd_dot16x2_ld64()
1296   static void qu8_gemm_3x4c8__wasmsimd_dot16x2_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_3x4c8__wasmsimd_dot16x2_ld64()  argument
1297     GEMMBenchmark(state,  in qu8_gemm_3x4c8__wasmsimd_dot16x2_ld64()
1302   static void qu8_gemm_4x4c8__wasmsimd_dot16x2_ld64(benchmark::State& state, const char* net) {  in qu8_gemm_4x4c8__wasmsimd_dot16x2_ld64()  argument
1303     GEMMBenchmark(state,  in qu8_gemm_4x4c8__wasmsimd_dot16x2_ld64()
1309   static void qu8_gemm_1x4c8__wasmsimd_dot16x2_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_1x4c8__wasmsimd_dot16x2_ld128()  argument
1310     GEMMBenchmark(state,  in qu8_gemm_1x4c8__wasmsimd_dot16x2_ld128()
1315   static void qu8_gemm_2x4c8__wasmsimd_dot16x2_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_2x4c8__wasmsimd_dot16x2_ld128()  argument
1316     GEMMBenchmark(state,  in qu8_gemm_2x4c8__wasmsimd_dot16x2_ld128()
1321   static void qu8_gemm_3x4c8__wasmsimd_dot16x2_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_3x4c8__wasmsimd_dot16x2_ld128()  argument
1322     GEMMBenchmark(state,  in qu8_gemm_3x4c8__wasmsimd_dot16x2_ld128()
1327   static void qu8_gemm_4x4c8__wasmsimd_dot16x2_ld128(benchmark::State& state, const char* net) {  in qu8_gemm_4x4c8__wasmsimd_dot16x2_ld128()  argument
1328     GEMMBenchmark(state,  in qu8_gemm_4x4c8__wasmsimd_dot16x2_ld128()
1364   static void qu8_gemm_1x2__wasm_fmagic(benchmark::State& state, const char* net) {  in BENCHMARK_GEMM()
1365     GEMMBenchmark(state,  in BENCHMARK_GEMM()
1370   static void qu8_gemm_2x2__wasm_fmagic(benchmark::State& state, const char* net) {  in qu8_gemm_2x2__wasm_fmagic()  argument
1371     GEMMBenchmark(state,  in qu8_gemm_2x2__wasm_fmagic()
1376   static void qu8_gemm_3x2__wasm_fmagic(benchmark::State& state, const char* net) {  in qu8_gemm_3x2__wasm_fmagic()  argument
1377     GEMMBenchmark(state,  in qu8_gemm_3x2__wasm_fmagic()
1382   static void qu8_gemm_4x2__wasm_fmagic(benchmark::State& state, const char* net) {  in qu8_gemm_4x2__wasm_fmagic()  argument
1383     GEMMBenchmark(state,  in qu8_gemm_4x2__wasm_fmagic()
1388   static void qu8_gemm_1x4__wasm_fmagic(benchmark::State& state, const char* net) {  in qu8_gemm_1x4__wasm_fmagic()  argument
1389     GEMMBenchmark(state,  in qu8_gemm_1x4__wasm_fmagic()
1394   static void qu8_gemm_2x4__wasm_fmagic(benchmark::State& state, const char* net) {  in qu8_gemm_2x4__wasm_fmagic()  argument
1395     GEMMBenchmark(state,  in qu8_gemm_2x4__wasm_fmagic()
1400   static void qu8_gemm_3x4__wasm_fmagic(benchmark::State& state, const char* net) {  in qu8_gemm_3x4__wasm_fmagic()  argument
1401     GEMMBenchmark(state,  in qu8_gemm_3x4__wasm_fmagic()
1406   static void qu8_gemm_4x4__wasm_fmagic(benchmark::State& state, const char* net) {  in qu8_gemm_4x4__wasm_fmagic()  argument
1407     GEMMBenchmark(state,  in qu8_gemm_4x4__wasm_fmagic()
1424 static void qu8_gemm_1x2__scalar_fmagic(benchmark::State& state, const char* net) {  in BENCHMARK_GEMM()
1425   GEMMBenchmark(state,  in BENCHMARK_GEMM()
1430 static void qu8_gemm_2x2__scalar_fmagic(benchmark::State& state, const char* net) {  in qu8_gemm_2x2__scalar_fmagic()  argument
1431   GEMMBenchmark(state,  in qu8_gemm_2x2__scalar_fmagic()
1436 static void qu8_gemm_3x2__scalar_fmagic(benchmark::State& state, const char* net) {  in qu8_gemm_3x2__scalar_fmagic()  argument
1437   GEMMBenchmark(state,  in qu8_gemm_3x2__scalar_fmagic()
1442 static void qu8_gemm_4x2__scalar_fmagic(benchmark::State& state, const char* net) {  in qu8_gemm_4x2__scalar_fmagic()  argument
1443   GEMMBenchmark(state,  in qu8_gemm_4x2__scalar_fmagic()
1448 static void qu8_gemm_1x4__scalar_fmagic(benchmark::State& state, const char* net) {  in qu8_gemm_1x4__scalar_fmagic()  argument
1449   GEMMBenchmark(state,  in qu8_gemm_1x4__scalar_fmagic()
1454 static void qu8_gemm_2x4__scalar_fmagic(benchmark::State& state, const char* net) {  in qu8_gemm_2x4__scalar_fmagic()  argument
1455   GEMMBenchmark(state,  in qu8_gemm_2x4__scalar_fmagic()
1460 static void qu8_gemm_3x4__scalar_fmagic(benchmark::State& state, const char* net) {  in qu8_gemm_3x4__scalar_fmagic()  argument
1461   GEMMBenchmark(state,  in qu8_gemm_3x4__scalar_fmagic()
1466 static void qu8_gemm_4x4__scalar_fmagic(benchmark::State& state, const char* net) {  in qu8_gemm_4x4__scalar_fmagic()  argument
1467   GEMMBenchmark(state,  in qu8_gemm_4x4__scalar_fmagic()
1473 static void qu8_gemm_1x2__scalar_imagic(benchmark::State& state, const char* net) {  in qu8_gemm_1x2__scalar_imagic()  argument
1474   GEMMBenchmark(state,  in qu8_gemm_1x2__scalar_imagic()
1479 static void qu8_gemm_2x2__scalar_imagic(benchmark::State& state, const char* net) {  in qu8_gemm_2x2__scalar_imagic()  argument
1480   GEMMBenchmark(state,  in qu8_gemm_2x2__scalar_imagic()
1485 static void qu8_gemm_3x2__scalar_imagic(benchmark::State& state, const char* net) {  in qu8_gemm_3x2__scalar_imagic()  argument
1486   GEMMBenchmark(state,  in qu8_gemm_3x2__scalar_imagic()
1491 static void qu8_gemm_4x2__scalar_imagic(benchmark::State& state, const char* net) {  in qu8_gemm_4x2__scalar_imagic()  argument
1492   GEMMBenchmark(state,  in qu8_gemm_4x2__scalar_imagic()
1497 static void qu8_gemm_1x4__scalar_imagic(benchmark::State& state, const char* net) {  in qu8_gemm_1x4__scalar_imagic()  argument
1498   GEMMBenchmark(state,  in qu8_gemm_1x4__scalar_imagic()
1503 static void qu8_gemm_2x4__scalar_imagic(benchmark::State& state, const char* net) {  in qu8_gemm_2x4__scalar_imagic()  argument
1504   GEMMBenchmark(state,  in qu8_gemm_2x4__scalar_imagic()
1509 static void qu8_gemm_3x4__scalar_imagic(benchmark::State& state, const char* net) {  in qu8_gemm_3x4__scalar_imagic()  argument
1510   GEMMBenchmark(state,  in qu8_gemm_3x4__scalar_imagic()
1515 static void qu8_gemm_4x4__scalar_imagic(benchmark::State& state, const char* net) {  in qu8_gemm_4x4__scalar_imagic()  argument
1516   GEMMBenchmark(state,  in qu8_gemm_4x4__scalar_imagic()
1522 static void qu8_gemm_1x2__scalar_lrintf(benchmark::State& state, const char* net) {  in qu8_gemm_1x2__scalar_lrintf()  argument
1523   GEMMBenchmark(state,  in qu8_gemm_1x2__scalar_lrintf()
1528 static void qu8_gemm_2x2__scalar_lrintf(benchmark::State& state, const char* net) {  in qu8_gemm_2x2__scalar_lrintf()  argument
1529   GEMMBenchmark(state,  in qu8_gemm_2x2__scalar_lrintf()
1534 static void qu8_gemm_3x2__scalar_lrintf(benchmark::State& state, const char* net) {  in qu8_gemm_3x2__scalar_lrintf()  argument
1535   GEMMBenchmark(state,  in qu8_gemm_3x2__scalar_lrintf()
1540 static void qu8_gemm_4x2__scalar_lrintf(benchmark::State& state, const char* net) {  in qu8_gemm_4x2__scalar_lrintf()  argument
1541   GEMMBenchmark(state,  in qu8_gemm_4x2__scalar_lrintf()
1546 static void qu8_gemm_1x4__scalar_lrintf(benchmark::State& state, const char* net) {  in qu8_gemm_1x4__scalar_lrintf()  argument
1547   GEMMBenchmark(state,  in qu8_gemm_1x4__scalar_lrintf()
1552 static void qu8_gemm_2x4__scalar_lrintf(benchmark::State& state, const char* net) {  in qu8_gemm_2x4__scalar_lrintf()  argument
1553   GEMMBenchmark(state,  in qu8_gemm_2x4__scalar_lrintf()
1558 static void qu8_gemm_3x4__scalar_lrintf(benchmark::State& state, const char* net) {  in qu8_gemm_3x4__scalar_lrintf()  argument
1559   GEMMBenchmark(state,  in qu8_gemm_3x4__scalar_lrintf()
1564 static void qu8_gemm_4x4__scalar_lrintf(benchmark::State& state, const char* net) {  in qu8_gemm_4x4__scalar_lrintf()  argument
1565   GEMMBenchmark(state,  in qu8_gemm_4x4__scalar_lrintf()