14 #include <benchmark/benchmark.h>
29 static void f32_igemm(benchmark::State& state,  in f32_igemm()
33   benchmark::utils::IsaCheckFunction isa_check = nullptr)  in f32_igemm()
65   const size_t mc_stride = benchmark::utils::RoundUp<size_t>(output_size, mr);  in f32_igemm()
66   const size_t nc_stride = benchmark::utils::RoundUp<size_t>(group_output_channels, nr);  in f32_igemm()
67   const size_t kc_stride = benchmark::utils::RoundUp<size_t>(group_input_channels, kr * sr);  in f32_igemm()
82     benchmark::utils::DivideRoundUp<size_t>(benchmark::utils::GetMaxCacheSize(),  in f32_igemm()
130     benchmark::utils::PrefetchToL1(a.data(), a.size() * sizeof(float));  in f32_igemm()
145   const uint64_t cpu_frequency = benchmark::utils::GetCurrentCpuFrequency();  in f32_igemm()
150   state.counters["FLOPS"] = benchmark::Counter(  in f32_igemm()
155     benchmark::Counter::kIsRate);  in f32_igemm()
159   static void f32_igemm(benchmark::State& state,  in f32_igemm()
163     benchmark::utils::IsaCheckFunction isa_check = nullptr)  in f32_igemm()
195   const size_t mc_stride = benchmark::utils::RoundUp<size_t>(output_size, mr);  in f32_igemm()
196   const size_t nc_stride = benchmark::utils::RoundUp<size_t>(group_output_channels, nr);  in f32_igemm()
197   const size_t kc_stride = benchmark::utils::RoundUp<size_t>(group_input_channels, kr * sr);  in f32_igemm()
212     benchmark::utils::DivideRoundUp<size_t>(benchmark::utils::GetMaxCacheSize(),  in f32_igemm()
279     benchmark::utils::PrefetchToL1(a.data(), a.size() * sizeof(float));  in f32_igemm()
295   const uint64_t cpu_frequency = benchmark::utils::GetCurrentCpuFrequency();  in f32_igemm()
300   state.counters["FLOPS"] = benchmark::Counter(  in f32_igemm()
305     benchmark::Counter::kIsRate);  in f32_igemm()
311 …static void jit_f32_igemm_1x8__aarch64_neonfma_cortex_a75(benchmark::State& state, const char* net…  in jit_f32_igemm_1x8__aarch64_neonfma_cortex_a75()
315 …static void jit_f32_igemm_1x8__aarch64_neonfma_prfm_cortex_a75(benchmark::State& state, const char…  in jit_f32_igemm_1x8__aarch64_neonfma_prfm_cortex_a75()
319 …static void jit_f32_igemm_4x8__aarch64_neonfma_cortex_a75(benchmark::State& state, const char* net…  in jit_f32_igemm_4x8__aarch64_neonfma_cortex_a75()
323 …static void jit_f32_igemm_4x8__aarch64_neonfma_prfm_cortex_a75(benchmark::State& state, const char…  in jit_f32_igemm_4x8__aarch64_neonfma_prfm_cortex_a75()
327   static void jit_f32_igemm_6x8__aarch64_neonfma_ld128(benchmark::State& state, const char* net) {  in jit_f32_igemm_6x8__aarch64_neonfma_ld128()
339   static void name(benchmark::State &state, const char *net) {                   \
357   static void jit_f32_igemm_4x8__aarch32_neon_ld64(benchmark::State& state, const char* net) {  in jit_f32_igemm_4x8__aarch32_neon_ld64()
361   static void jit_f32_igemm_4x8__aarch32_neon_cortex_a7(benchmark::State& state, const char* net) {  in jit_f32_igemm_4x8__aarch32_neon_cortex_a7()
365   static void jit_f32_igemm_4x8__aarch32_neon_cortex_a53(benchmark::State& state, const char* net) {  in jit_f32_igemm_4x8__aarch32_neon_cortex_a53()
369   static void jit_f32_igemm_4x8__aarch32_neon_cortex_a55(benchmark::State& state, const char* net) {  in jit_f32_igemm_4x8__aarch32_neon_cortex_a55()
373 …static void jit_f32_igemm_4x8__aarch32_neon_prfm_cortex_a75(benchmark::State& state, const char* n…  in jit_f32_igemm_4x8__aarch32_neon_prfm_cortex_a75()
377   static void jit_f32_igemm_4x8__aarch32_neon_cortex_a75(benchmark::State& state, const char* net) {  in jit_f32_igemm_4x8__aarch32_neon_cortex_a75()
391   static void f32_igemm_4x8__aarch32_neon_ld64(benchmark::State& state, const char* net) {  in BENCHMARK_CONV()
395   static void f32_igemm_4x8__aarch32_neon_cortex_a7(benchmark::State& state, const char* net) {  in f32_igemm_4x8__aarch32_neon_cortex_a7()
399   static void f32_igemm_4x8__aarch32_neon_cortex_a53(benchmark::State& state, const char* net) {  in f32_igemm_4x8__aarch32_neon_cortex_a53()
403 …static void f32_igemm_4x8__aarch32_neon_prfm_cortex_a53(benchmark::State& state, const char* net) {  in f32_igemm_4x8__aarch32_neon_prfm_cortex_a53()
407   static void f32_igemm_4x8__aarch32_neon_cortex_a55(benchmark::State& state, const char* net) {  in f32_igemm_4x8__aarch32_neon_cortex_a55()
411 …static void f32_igemm_4x8__aarch32_neon_prfm_cortex_a75(benchmark::State& state, const char* net) {  in f32_igemm_4x8__aarch32_neon_prfm_cortex_a75()
415   static void f32_igemm_4x8__aarch32_neon_cortex_a75(benchmark::State& state, const char* net) {  in f32_igemm_4x8__aarch32_neon_cortex_a75()
431   static void f32_igemm_1x12__aarch64_neonfma_cortex_a53(benchmark::State& state, const char* net) {  in BENCHMARK_CONV()
435   static void f32_igemm_1x8__aarch64_neonfma_cortex_a53(benchmark::State& state, const char* net) {  in f32_igemm_1x8__aarch64_neonfma_cortex_a53()
439 …static void f32_igemm_1x8__aarch64_neonfma_prfm_cortex_a53(benchmark::State& state, const char* ne…  in f32_igemm_1x8__aarch64_neonfma_prfm_cortex_a53()
443   static void f32_igemm_1x8__aarch64_neonfma_cortex_a75(benchmark::State& state, const char* net) {  in f32_igemm_1x8__aarch64_neonfma_cortex_a75()
447 …static void f32_igemm_1x8__aarch64_neonfma_prfm_cortex_a75(benchmark::State& state, const char* ne…  in f32_igemm_1x8__aarch64_neonfma_prfm_cortex_a75()
451   static void f32_igemm_4x2__aarch64_neonfma_cortex_a75(benchmark::State& state, const char* net) {  in f32_igemm_4x2__aarch64_neonfma_cortex_a75()
455 …static void f32_igemm_4x2__aarch64_neonfma_prfm_cortex_a75(benchmark::State& state, const char* ne…  in f32_igemm_4x2__aarch64_neonfma_prfm_cortex_a75()
459   static void f32_igemm_4x2__aarch64_neonfma_ld64(benchmark::State& state, const char* net) {  in f32_igemm_4x2__aarch64_neonfma_ld64()
463   static void f32_igemm_4x8__aarch64_neonfma_cortex_a53(benchmark::State& state, const char* net) {  in f32_igemm_4x8__aarch64_neonfma_cortex_a53()
467 …static void f32_igemm_4x8__aarch64_neonfma_prfm_cortex_a53(benchmark::State& state, const char* ne…  in f32_igemm_4x8__aarch64_neonfma_prfm_cortex_a53()
471   static void f32_igemm_4x8__aarch64_neonfma_cortex_a55(benchmark::State& state, const char* net) {  in f32_igemm_4x8__aarch64_neonfma_cortex_a55()
475   static void f32_igemm_4x8__aarch64_neonfma_cortex_a75(benchmark::State& state, const char* net) {  in f32_igemm_4x8__aarch64_neonfma_cortex_a75()
479 …static void f32_igemm_4x8__aarch64_neonfma_prfm_cortex_a75(benchmark::State& state, const char* ne…  in f32_igemm_4x8__aarch64_neonfma_prfm_cortex_a75()
483   static void f32_igemm_4x8__aarch64_neonfma_ld64(benchmark::State& state, const char* net) {  in f32_igemm_4x8__aarch64_neonfma_ld64()
487   static void f32_igemm_4x8__aarch64_neonfma_ld128(benchmark::State& state, const char* net) {  in f32_igemm_4x8__aarch64_neonfma_ld128()
491   static void f32_igemm_5x8__aarch64_neonfma_cortex_a75(benchmark::State& state, const char* net) {  in f32_igemm_5x8__aarch64_neonfma_cortex_a75()
495 …static void f32_igemm_5x8__aarch64_neonfma_prfm_cortex_a75(benchmark::State& state, const char* ne…  in f32_igemm_5x8__aarch64_neonfma_prfm_cortex_a75()
499   static void f32_igemm_4x12__aarch64_neonfma_cortex_a53(benchmark::State& state, const char* net) {  in f32_igemm_4x12__aarch64_neonfma_cortex_a53()
503   static void f32_igemm_6x8__aarch64_neonfma_cortex_a53(benchmark::State& state, const char* net) {  in f32_igemm_6x8__aarch64_neonfma_cortex_a53()
507 …static void f32_igemm_6x8__aarch64_neonfma_prfm_cortex_a53(benchmark::State& state, const char* ne…  in f32_igemm_6x8__aarch64_neonfma_prfm_cortex_a53()
511   static void f32_igemm_6x8__aarch64_neonfma_cortex_a55(benchmark::State& state, const char* net) {  in f32_igemm_6x8__aarch64_neonfma_cortex_a55()
515   static void f32_igemm_6x8__aarch64_neonfma_cortex_a73(benchmark::State& state, const char* net) {  in f32_igemm_6x8__aarch64_neonfma_cortex_a73()
519   static void f32_igemm_6x8__aarch64_neonfma_cortex_a75(benchmark::State& state, const char* net) {  in f32_igemm_6x8__aarch64_neonfma_cortex_a75()
523 …static void f32_igemm_6x8__aarch64_neonfma_prfm_cortex_a75(benchmark::State& state, const char* ne…  in f32_igemm_6x8__aarch64_neonfma_prfm_cortex_a75()
527   static void f32_igemm_6x8__aarch64_neonfma_ld64(benchmark::State& state, const char* net) {  in f32_igemm_6x8__aarch64_neonfma_ld64()
531   static void f32_igemm_6x8__aarch64_neonfma_ld128(benchmark::State& state, const char* net) {  in f32_igemm_6x8__aarch64_neonfma_ld128()
535   static void f32_igemm_1x8__neonfma_lane_ld64(benchmark::State& state, const char* net) {  in f32_igemm_1x8__neonfma_lane_ld64()
539   static void f32_igemm_4x2__neonfma_lane_ld64(benchmark::State& state, const char* net) {  in f32_igemm_4x2__neonfma_lane_ld64()
543   static void f32_igemm_6x2__neonfma_lane_ld64(benchmark::State& state, const char* net) {  in f32_igemm_6x2__neonfma_lane_ld64()
547   static void f32_igemm_4x4__neonfma_lane_ld64(benchmark::State& state, const char* net) {  in f32_igemm_4x4__neonfma_lane_ld64()
551   static void f32_igemm_4x8__neonfma_lane_ld128(benchmark::State& state, const char* net) {  in f32_igemm_4x8__neonfma_lane_ld128()
555   static void f32_igemm_4x8__neonfma_lane_ld64(benchmark::State& state, const char* net) {  in f32_igemm_4x8__neonfma_lane_ld64()
559   static void f32_igemm_6x8__neonfma_lane_ld64(benchmark::State& state, const char* net) {  in f32_igemm_6x8__neonfma_lane_ld64()
563   static void f32_igemm_6x8__neonfma_lane_ld128(benchmark::State& state, const char* net) {  in f32_igemm_6x8__neonfma_lane_ld128()
606   static void f32_igemm_1x8__neon_lane_ld64(benchmark::State& state, const char* net) {  in BENCHMARK_CONV()
608       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckNEON);  in BENCHMARK_CONV()
610   static void f32_igemm_4x2__neon_lane_ld64(benchmark::State& state, const char* net) {  in f32_igemm_4x2__neon_lane_ld64()
612       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckNEON);  in f32_igemm_4x2__neon_lane_ld64()
614   static void f32_igemm_6x2__neon_lane_ld64(benchmark::State& state, const char* net) {  in f32_igemm_6x2__neon_lane_ld64()
616       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckNEON);  in f32_igemm_6x2__neon_lane_ld64()
618   static void f32_igemm_4x4__neon_lane_ld64(benchmark::State& state, const char* net) {  in f32_igemm_4x4__neon_lane_ld64()
620       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckNEON);  in f32_igemm_4x4__neon_lane_ld64()
622   static void f32_igemm_4x8__neon_lane_ld64(benchmark::State& state, const char* net) {  in f32_igemm_4x8__neon_lane_ld64()
624       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckNEON);  in f32_igemm_4x8__neon_lane_ld64()
626   static void f32_igemm_4x8__neon_lane_ld128(benchmark::State& state, const char* net) {  in f32_igemm_4x8__neon_lane_ld128()
628       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckNEON);  in f32_igemm_4x8__neon_lane_ld128()
630   static void f32_igemm_6x8__neon_lane_ld64(benchmark::State& state, const char* net) {  in f32_igemm_6x8__neon_lane_ld64()
632       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckNEON);  in f32_igemm_6x8__neon_lane_ld64()
634   static void f32_igemm_6x8__neon_lane_ld128(benchmark::State& state, const char* net) {  in f32_igemm_6x8__neon_lane_ld128()
636       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckNEON);  in f32_igemm_6x8__neon_lane_ld128()
638   static void f32_igemm_1x8__neon_dup_ld64(benchmark::State& state, const char* net) {  in f32_igemm_1x8__neon_dup_ld64()
640       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckNEON);  in f32_igemm_1x8__neon_dup_ld64()
642   static void f32_igemm_4x8__neon_dup_ld128(benchmark::State& state, const char* net) {  in f32_igemm_4x8__neon_dup_ld128()
644       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckNEON);  in f32_igemm_4x8__neon_dup_ld128()
646   static void f32_igemm_4x8__neon_dup_ld64(benchmark::State& state, const char* net) {  in f32_igemm_4x8__neon_dup_ld64()
648       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckNEON);  in f32_igemm_4x8__neon_dup_ld64()
650   static void f32_igemm_6x8__neon_dup_ld64(benchmark::State& state, const char* net) {  in f32_igemm_6x8__neon_dup_ld64()
652       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckNEON);  in f32_igemm_6x8__neon_dup_ld64()
654   static void f32_igemm_6x8__neon_dup_ld128(benchmark::State& state, const char* net) {  in f32_igemm_6x8__neon_dup_ld128()
656       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckNEON);  in f32_igemm_6x8__neon_dup_ld128()
658   static void f32_igemm_1x8__neonfma_dup_ld64(benchmark::State& state, const char* net) {  in f32_igemm_1x8__neonfma_dup_ld64()
660       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckNEONFMA);  in f32_igemm_1x8__neonfma_dup_ld64()
662   static void f32_igemm_4x8__neonfma_dup_ld128(benchmark::State& state, const char* net) {  in f32_igemm_4x8__neonfma_dup_ld128()
664       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckNEONFMA);  in f32_igemm_4x8__neonfma_dup_ld128()
666   static void f32_igemm_4x8__neonfma_dup_ld64(benchmark::State& state, const char* net) {  in f32_igemm_4x8__neonfma_dup_ld64()
668       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckNEONFMA);  in f32_igemm_4x8__neonfma_dup_ld64()
670   static void f32_igemm_6x8__neonfma_dup_ld64(benchmark::State& state, const char* net) {  in f32_igemm_6x8__neonfma_dup_ld64()
672       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckNEONFMA);  in f32_igemm_6x8__neonfma_dup_ld64()
674   static void f32_igemm_6x8__neonfma_dup_ld128(benchmark::State& state, const char* net) {  in f32_igemm_6x8__neonfma_dup_ld128()
676       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckNEONFMA);  in f32_igemm_6x8__neonfma_dup_ld128()
678   static void f32_igemm_1x8s4__neon(benchmark::State& state, const char* net) {  in f32_igemm_1x8s4__neon()
680       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckNEON);  in f32_igemm_1x8s4__neon()
682   static void f32_igemm_4x8s4__neon(benchmark::State& state, const char* net) {  in f32_igemm_4x8s4__neon()
684       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckNEON);  in f32_igemm_4x8s4__neon()
686   static void f32_igemm_6x8s4__neon(benchmark::State& state, const char* net) {  in f32_igemm_6x8s4__neon()
688       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckNEON);  in f32_igemm_6x8s4__neon()
690   static void f32_igemm_8x8s4__neon(benchmark::State& state, const char* net) {  in f32_igemm_8x8s4__neon()
692       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckNEON);  in f32_igemm_8x8s4__neon()
694   static void f32_igemm_1x8s4__neonfma(benchmark::State& state, const char* net) {  in f32_igemm_1x8s4__neonfma()
696       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckNEONFMA);  in f32_igemm_1x8s4__neonfma()
698   static void f32_igemm_4x8s4__neonfma(benchmark::State& state, const char* net) {  in f32_igemm_4x8s4__neonfma()
700       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckNEONFMA);  in f32_igemm_4x8s4__neonfma()
702   static void f32_igemm_6x8s4__neonfma(benchmark::State& state, const char* net) {  in f32_igemm_6x8s4__neonfma()
704       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckNEONFMA);  in f32_igemm_6x8s4__neonfma()
706   static void f32_igemm_8x8s4__neonfma(benchmark::State& state, const char* net) {  in f32_igemm_8x8s4__neonfma()
708       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckNEONFMA);  in f32_igemm_8x8s4__neonfma()
742   static void f32_igemm_1x8__sse_load1(benchmark::State& state, const char* net) {  in BENCHMARK_CONV()
746   static void f32_igemm_3x8__sse_load1(benchmark::State& state, const char* net) {  in f32_igemm_3x8__sse_load1()
750   static void f32_igemm_4x8__sse_load1(benchmark::State& state, const char* net) {  in f32_igemm_4x8__sse_load1()
754   static void f32_igemm_5x8__sse_load1(benchmark::State& state, const char* net) {  in f32_igemm_5x8__sse_load1()
759   static void f32_igemm_1x8__sse_dup(benchmark::State& state, const char* net) {  in f32_igemm_1x8__sse_dup()
763   static void f32_igemm_3x8__sse_dup(benchmark::State& state, const char* net) {  in f32_igemm_3x8__sse_dup()
767   static void f32_igemm_4x8__sse_dup(benchmark::State& state, const char* net) {  in f32_igemm_4x8__sse_dup()
771   static void f32_igemm_5x8__sse_dup(benchmark::State& state, const char* net) {  in f32_igemm_5x8__sse_dup()
776   static void f32_igemm_1x8s4__sse(benchmark::State& state, const char* net) {  in f32_igemm_1x8s4__sse()
780   static void f32_igemm_3x8s4__sse(benchmark::State& state, const char* net) {  in f32_igemm_3x8s4__sse()
784   static void f32_igemm_4x8s4__sse(benchmark::State& state, const char* net) {  in f32_igemm_4x8s4__sse()
788   static void f32_igemm_5x8s4__sse(benchmark::State& state, const char* net) {  in f32_igemm_5x8s4__sse()
793   static void f32_igemm_1x8__sse2_dup(benchmark::State& state, const char* net) {  in f32_igemm_1x8__sse2_dup()
797   static void f32_igemm_3x8__sse2_dup(benchmark::State& state, const char* net) {  in f32_igemm_3x8__sse2_dup()
801   static void f32_igemm_4x8__sse2_dup(benchmark::State& state, const char* net) {  in f32_igemm_4x8__sse2_dup()
805   static void f32_igemm_5x8__sse2_dup(benchmark::State& state, const char* net) {  in f32_igemm_5x8__sse2_dup()
810   static void f32_igemm_1x8__avx_broadcast(benchmark::State& state, const char* net) {  in f32_igemm_1x8__avx_broadcast()
812       xnn_init_f32_minmax_avx_params, benchmark::utils::CheckAVX);  in f32_igemm_1x8__avx_broadcast()
814   static void f32_igemm_4x8__avx_broadcast(benchmark::State& state, const char* net) {  in f32_igemm_4x8__avx_broadcast()
816       xnn_init_f32_minmax_avx_params, benchmark::utils::CheckAVX);  in f32_igemm_4x8__avx_broadcast()
818   static void f32_igemm_5x8__avx_broadcast(benchmark::State& state, const char* net) {  in f32_igemm_5x8__avx_broadcast()
820       xnn_init_f32_minmax_avx_params, benchmark::utils::CheckAVX);  in f32_igemm_5x8__avx_broadcast()
822   static void f32_igemm_6x8__avx_broadcast(benchmark::State& state, const char* net) {  in f32_igemm_6x8__avx_broadcast()
824       xnn_init_f32_minmax_avx_params, benchmark::utils::CheckAVX);  in f32_igemm_6x8__avx_broadcast()
826   static void f32_igemm_7x8__avx_broadcast(benchmark::State& state, const char* net) {  in f32_igemm_7x8__avx_broadcast()
828       xnn_init_f32_minmax_avx_params, benchmark::utils::CheckAVX);  in f32_igemm_7x8__avx_broadcast()
831   static void f32_igemm_1x8__fma3_broadcast(benchmark::State& state, const char* net) {  in f32_igemm_1x8__fma3_broadcast()
833       xnn_init_f32_minmax_avx_params, benchmark::utils::CheckFMA3);  in f32_igemm_1x8__fma3_broadcast()
835   static void f32_igemm_4x8__fma3_broadcast(benchmark::State& state, const char* net) {  in f32_igemm_4x8__fma3_broadcast()
837       xnn_init_f32_minmax_avx_params, benchmark::utils::CheckFMA3);  in f32_igemm_4x8__fma3_broadcast()
839   static void f32_igemm_5x8__fma3_broadcast(benchmark::State& state, const char* net) {  in f32_igemm_5x8__fma3_broadcast()
841       xnn_init_f32_minmax_avx_params, benchmark::utils::CheckFMA3);  in f32_igemm_5x8__fma3_broadcast()
843   static void f32_igemm_6x8__fma3_broadcast(benchmark::State& state, const char* net) {  in f32_igemm_6x8__fma3_broadcast()
845       xnn_init_f32_minmax_avx_params, benchmark::utils::CheckFMA3);  in f32_igemm_6x8__fma3_broadcast()
847   static void f32_igemm_7x8__fma3_broadcast(benchmark::State& state, const char* net) {  in f32_igemm_7x8__fma3_broadcast()
849       xnn_init_f32_minmax_avx_params, benchmark::utils::CheckFMA3);  in f32_igemm_7x8__fma3_broadcast()
851   static void f32_igemm_8x8__fma3_broadcast(benchmark::State& state, const char* net) {  in f32_igemm_8x8__fma3_broadcast()
853       xnn_init_f32_minmax_avx_params, benchmark::utils::CheckFMA3);  in f32_igemm_8x8__fma3_broadcast()
856   static void f32_igemm_1x16__avx512f_broadcast(benchmark::State& state, const char* net) {  in f32_igemm_1x16__avx512f_broadcast()
858       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckFMA3);  in f32_igemm_1x16__avx512f_broadcast()
860   static void f32_igemm_4x16__avx512f_broadcast(benchmark::State& state, const char* net) {  in f32_igemm_4x16__avx512f_broadcast()
862       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckFMA3);  in f32_igemm_4x16__avx512f_broadcast()
864   static void f32_igemm_5x16__avx512f_broadcast(benchmark::State& state, const char* net) {  in f32_igemm_5x16__avx512f_broadcast()
866       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckFMA3);  in f32_igemm_5x16__avx512f_broadcast()
868   static void f32_igemm_6x16__avx512f_broadcast(benchmark::State& state, const char* net) {  in f32_igemm_6x16__avx512f_broadcast()
870       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckFMA3);  in f32_igemm_6x16__avx512f_broadcast()
872   static void f32_igemm_7x16__avx512f_broadcast(benchmark::State& state, const char* net) {  in f32_igemm_7x16__avx512f_broadcast()
874       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckFMA3);  in f32_igemm_7x16__avx512f_broadcast()
876   static void f32_igemm_8x16__avx512f_broadcast(benchmark::State& state, const char* net) {  in f32_igemm_8x16__avx512f_broadcast()
878       xnn_init_f32_minmax_scalar_params, benchmark::utils::CheckFMA3);  in f32_igemm_8x16__avx512f_broadcast()
924   static void f32_igemm_3x8__wasmsimd_arm_loadsplat(benchmark::State& state, const char* net) {  in BENCHMARK_CONV()
929   static void f32_igemm_4x8__wasmsimd_arm_loadsplat(benchmark::State& state, const char* net) {  in f32_igemm_4x8__wasmsimd_arm_loadsplat()
934   static void f32_igemm_5x8__wasmsimd_arm_loadsplat(benchmark::State& state, const char* net) {  in f32_igemm_5x8__wasmsimd_arm_loadsplat()
939   static void f32_igemm_6x8__wasmsimd_arm_loadsplat(benchmark::State& state, const char* net) {  in f32_igemm_6x8__wasmsimd_arm_loadsplat()
944   static void f32_igemm_3x8__wasmsimd_x86_loadsplat(benchmark::State& state, const char* net) {  in f32_igemm_3x8__wasmsimd_x86_loadsplat()
949   static void f32_igemm_4x8__wasmsimd_x86_loadsplat(benchmark::State& state, const char* net) {  in f32_igemm_4x8__wasmsimd_x86_loadsplat()
954   static void f32_igemm_5x8__wasmsimd_x86_loadsplat(benchmark::State& state, const char* net) {  in f32_igemm_5x8__wasmsimd_x86_loadsplat()
959   static void f32_igemm_6x8__wasmsimd_x86_loadsplat(benchmark::State& state, const char* net) {  in f32_igemm_6x8__wasmsimd_x86_loadsplat()
964   static void f32_igemm_3x8__wasmsimd_arm_splat(benchmark::State& state, const char* net) {  in f32_igemm_3x8__wasmsimd_arm_splat()
969   static void f32_igemm_4x8__wasmsimd_arm_splat(benchmark::State& state, const char* net) {  in f32_igemm_4x8__wasmsimd_arm_splat()
974   static void f32_igemm_5x8__wasmsimd_arm_splat(benchmark::State& state, const char* net) {  in f32_igemm_5x8__wasmsimd_arm_splat()
979   static void f32_igemm_6x8__wasmsimd_arm_splat(benchmark::State& state, const char* net) {  in f32_igemm_6x8__wasmsimd_arm_splat()
984   static void f32_igemm_3x8__wasmsimd_x86_splat(benchmark::State& state, const char* net) {  in f32_igemm_3x8__wasmsimd_x86_splat()
989   static void f32_igemm_4x8__wasmsimd_x86_splat(benchmark::State& state, const char* net) {  in f32_igemm_4x8__wasmsimd_x86_splat()
994   static void f32_igemm_5x8__wasmsimd_x86_splat(benchmark::State& state, const char* net) {  in f32_igemm_5x8__wasmsimd_x86_splat()
999   static void f32_igemm_6x8__wasmsimd_x86_splat(benchmark::State& state, const char* net) {  in f32_igemm_6x8__wasmsimd_x86_splat()
1004   static void f32_igemm_3x8s4__wasmsimd_arm(benchmark::State& state, const char* net) {  in f32_igemm_3x8s4__wasmsimd_arm()
1009   static void f32_igemm_4x8s4__wasmsimd_arm(benchmark::State& state, const char* net) {  in f32_igemm_4x8s4__wasmsimd_arm()
1014   static void f32_igemm_5x8s4__wasmsimd_arm(benchmark::State& state, const char* net) {  in f32_igemm_5x8s4__wasmsimd_arm()
1019   static void f32_igemm_6x8s4__wasmsimd_arm(benchmark::State& state, const char* net) {  in f32_igemm_6x8s4__wasmsimd_arm()
1024   static void f32_igemm_3x8s4__wasmsimd_x86(benchmark::State& state, const char* net) {  in f32_igemm_3x8s4__wasmsimd_x86()
1029   static void f32_igemm_4x8s4__wasmsimd_x86(benchmark::State& state, const char* net) {  in f32_igemm_4x8s4__wasmsimd_x86()
1034   static void f32_igemm_5x8s4__wasmsimd_x86(benchmark::State& state, const char* net) {  in f32_igemm_5x8s4__wasmsimd_x86()
1039   static void f32_igemm_6x8s4__wasmsimd_x86(benchmark::State& state, const char* net) {  in f32_igemm_6x8s4__wasmsimd_x86()
1071 static void f32_igemm_1x4__scalar(benchmark::State& state, const char* net) {  in BENCHMARK_CONV()
1076 static void f32_igemm_2x4__scalar(benchmark::State& state, const char* net) {  in f32_igemm_2x4__scalar()
1081 static void f32_igemm_4x4__scalar(benchmark::State& state, const char* net) {  in f32_igemm_4x4__scalar()