1/* 2 * Copyright (c) 2013-2022, Arm Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7#include <arch.h> 8#include <asm_macros.S> 9 10 .globl flush_dcache_range 11 .globl flush_dcache_to_popa_range 12 .globl flush_dcache_to_popa_range_mte2 13 .globl clean_dcache_range 14 .globl inv_dcache_range 15 .globl dcsw_op_louis 16 .globl dcsw_op_all 17 .globl dcsw_op_level1 18 .globl dcsw_op_level2 19 .globl dcsw_op_level3 20 21/* Opcodes for data cache maintenance by PA instructions. */ 22 23/* 24 * sys #6, c7, c14, #1, x0 25 * DC CIPAPA, X0 26 */ 27#define dc_cipapa_x0 0xd50e7e20 28 29/* 30 * sys #6, c7, c14, #3, x0 31 * DC CIDGPAPA, X0 32 */ 33#define dc_cigdpapa_x0 0xd50e7ea0 34 35/* 36 * This macro can be used for implementing various data cache operations `op` 37 */ 38.macro do_dcache_maintenance_by_mva op 39 /* Exit early if size is zero */ 40 cbz x1, exit_loop_\op 41 dcache_line_size x2, x3 42 add x1, x0, x1 43 sub x3, x2, #1 44 bic x0, x0, x3 45loop_\op: 46 dc \op, x0 47 add x0, x0, x2 48 cmp x0, x1 49 b.lo loop_\op 50 dsb sy 51exit_loop_\op: 52 ret 53.endm 54 55/* op: the hexadecimal instruction opcode for the cache operation */ 56.macro do_dcache_maintenance_instr op 57 /* Exit early if size is zero */ 58 cbz x1, exit_loop_\op 59 dcache_line_size x2, x3 60 sub x3, x2, #1 61 bic x0, x0, x3 62 add x1, x1, x0 63loop_\op: 64 .inst \op 65 add x0, x0, x2 66 cmp x0, x1 67 b.lo loop_\op 68 dsb osh 69exit_loop_\op: 70 ret 71.endm 72 73.macro check_plat_can_cmo 74#if CONDITIONAL_CMO 75 mov x3, x30 76 mov x2, x0 77 bl plat_can_cmo 78 mov x30, x3 79 cbnz x0, 1f 80 ret 811: 82 mov x0, x2 83#endif 84.endm 85 86 /* ------------------------------------------- 87 * DCache Clean+Invalidate by MVA from base 88 * address till size. 'x0' = addr, 'x1' = size 89 * ------------------------------------------- 90 */ 91func flush_dcache_range 92 check_plat_can_cmo 93 do_dcache_maintenance_by_mva civac 94endfunc flush_dcache_range 95 96 /* ------------------------------------------ 97 * DCache Clean by MVA from base address till 98 * size. 'x0' = addr, 'x1' = size 99 * ------------------------------------------ 100 */ 101func clean_dcache_range 102 check_plat_can_cmo 103 do_dcache_maintenance_by_mva cvac 104endfunc clean_dcache_range 105 106 /* ------------------------------------------ 107 * DCache Invalidate by MVA from base address 108 * till size. 'x0' = addr, 'x1' = size 109 * ------------------------------------------ 110 */ 111func inv_dcache_range 112 check_plat_can_cmo 113 do_dcache_maintenance_by_mva ivac 114endfunc inv_dcache_range 115 116 /* 117 * ------------------------------------------ 118 * DCache Clean+Invalidate by PA to POPA from 119 * base address till size. 120 * 'x0' = addr, 'x1' = size 121 * ------------------------------------------ 122 */ 123func flush_dcache_to_popa_range 124 check_plat_can_cmo 125 /* dc cipapa, x0 */ 126 do_dcache_maintenance_instr dc_cipapa_x0 127endfunc flush_dcache_to_popa_range 128 129 /* 130 * ------------------------------------------ 131 * Clean+Invalidate by PA to POPA (MTE2) 132 * from base address till size. 133 * 'x0' = addr, 'x1' = size 134 * ------------------------------------------ 135 * On implementations with FEAT_MTE2, Root firmware must issue 136 * DC_CIGDPAPA instead of DC_CIPAPA, in order to additionally 137 * clean and invalidate Allocation Tags associated with the 138 * affected locations. 139 */ 140func flush_dcache_to_popa_range_mte2 141 check_plat_can_cmo 142 /* dc cigdpapa, x0 */ 143 do_dcache_maintenance_instr dc_cigdpapa_x0 144endfunc flush_dcache_to_popa_range_mte2 145 146 /* --------------------------------------------------------------- 147 * Data cache operations by set/way to the level specified 148 * 149 * The main function, do_dcsw_op requires: 150 * x0: The operation type (0-2), as defined in arch.h 151 * x3: The last cache level to operate on 152 * x9: clidr_el1 153 * x10: The cache level to begin operation from 154 * and will carry out the operation on each data cache from level 0 155 * to the level in x3 in sequence 156 * 157 * The dcsw_op macro sets up the x3 and x9 parameters based on 158 * clidr_el1 cache information before invoking the main function 159 * --------------------------------------------------------------- 160 */ 161 162 .macro dcsw_op shift, fw, ls 163 mrs x9, clidr_el1 164 ubfx x3, x9, \shift, \fw 165 lsl x3, x3, \ls 166 mov x10, xzr 167 b do_dcsw_op 168 .endm 169 170func do_dcsw_op 171 cbz x3, exit 172 mrs x12, ID_AA64MMFR2_EL1 // stash FEAT_CCIDX identifier in x12 173 ubfx x12, x12, #ID_AA64MMFR2_EL1_CCIDX_SHIFT, #ID_AA64MMFR2_EL1_CCIDX_LENGTH 174 adr x14, dcsw_loop_table // compute inner loop address 175 add x14, x14, x0, lsl #5 // inner loop is 8x32-bit instructions 176#if ENABLE_BTI 177 add x14, x14, x0, lsl #2 // inner loop is + "bti j" instruction 178#endif 179 mov x0, x9 180 mov w8, #1 181loop1: 182 add x2, x10, x10, lsr #1 // work out 3x current cache level 183 lsr x1, x0, x2 // extract cache type bits from clidr 184 and x1, x1, #7 // mask the bits for current cache only 185 cmp x1, #2 // see what cache we have at this level 186 b.lo level_done // nothing to do if no cache or icache 187 188 msr csselr_el1, x10 // select current cache level in csselr 189 isb // isb to sych the new cssr&csidr 190 mrs x1, ccsidr_el1 // read the new ccsidr 191 and x2, x1, #7 // extract the length of the cache lines 192 add x2, x2, #4 // add 4 (line length offset) 193 194 cbz x12, 1f // check for FEAT_CCIDX for Associativity 195 ubfx x4, x1, #3, #21 // x4 = associativity CCSIDR_EL1[23:3] 196 b 2f 1971: 198 ubfx x4, x1, #3, #10 // x4 = associativity CCSIDR_EL1[12:3] 1992: 200 clz w5, w4 // bit position of way size increment 201 lsl w9, w4, w5 // w9 = aligned max way number 202 lsl w16, w8, w5 // w16 = way number loop decrement 203 orr w9, w10, w9 // w9 = combine way and cache number 204 205 cbz x12, 3f // check for FEAT_CCIDX for NumSets 206 ubfx x6, x1, #32, #24 // x6 (w6) = numsets CCSIDR_EL1[55:32] 207 // ISA will not allow x->w ubfx 208 b 4f 2093: 210 ubfx w6, w1, #13, #15 // w6 = numsets CCSIDR_EL1[27:13] 2114: 212 lsl w17, w8, w2 // w17 = set number loop decrement 213 dsb sy // barrier before we start this level 214 br x14 // jump to DC operation specific loop 215 216 .macro dcsw_loop _op 217#if ENABLE_BTI 218 bti j 219#endif 220loop2_\_op: 221 lsl w7, w6, w2 // w7 = aligned max set number 222 223loop3_\_op: 224 orr w11, w9, w7 // combine cache, way and set number 225 dc \_op, x11 226 subs w7, w7, w17 // decrement set number 227 b.hs loop3_\_op 228 229 subs x9, x9, x16 // decrement way number 230 b.hs loop2_\_op 231 232 b level_done 233 .endm 234 235level_done: 236 add x10, x10, #2 // increment cache number 237 cmp x3, x10 238 b.hi loop1 239 msr csselr_el1, xzr // select cache level 0 in csselr 240 dsb sy // barrier to complete final cache operation 241 isb 242exit: 243 ret 244endfunc do_dcsw_op 245 246dcsw_loop_table: 247 dcsw_loop isw 248 dcsw_loop cisw 249 dcsw_loop csw 250 251 252func dcsw_op_louis 253 check_plat_can_cmo 254 dcsw_op #LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT 255endfunc dcsw_op_louis 256 257 258func dcsw_op_all 259 check_plat_can_cmo 260 dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT 261endfunc dcsw_op_all 262 263 /* --------------------------------------------------------------- 264 * Helper macro for data cache operations by set/way for the 265 * level specified 266 * --------------------------------------------------------------- 267 */ 268 .macro dcsw_op_level level 269 mrs x9, clidr_el1 270 mov x3, \level 271 sub x10, x3, #2 272 b do_dcsw_op 273 .endm 274 275 /* --------------------------------------------------------------- 276 * Data cache operations by set/way for level 1 cache 277 * 278 * The main function, do_dcsw_op requires: 279 * x0: The operation type (0-2), as defined in arch.h 280 * --------------------------------------------------------------- 281 */ 282func dcsw_op_level1 283 check_plat_can_cmo 284 dcsw_op_level #(1 << LEVEL_SHIFT) 285endfunc dcsw_op_level1 286 287 /* --------------------------------------------------------------- 288 * Data cache operations by set/way for level 2 cache 289 * 290 * The main function, do_dcsw_op requires: 291 * x0: The operation type (0-2), as defined in arch.h 292 * --------------------------------------------------------------- 293 */ 294func dcsw_op_level2 295 check_plat_can_cmo 296 dcsw_op_level #(2 << LEVEL_SHIFT) 297endfunc dcsw_op_level2 298 299 /* --------------------------------------------------------------- 300 * Data cache operations by set/way for level 3 cache 301 * 302 * The main function, do_dcsw_op requires: 303 * x0: The operation type (0-2), as defined in arch.h 304 * --------------------------------------------------------------- 305 */ 306func dcsw_op_level3 307 check_plat_can_cmo 308 dcsw_op_level #(3 << LEVEL_SHIFT) 309endfunc dcsw_op_level3 310