1/* SPDX-License-Identifier: GPL-2.0-only */ 2 3#include <commonlib/helpers.h> 4#include <cpu/intel/msr.h> 5#include <cpu/x86/cache.h> 6#include <cpu/x86/cr.h> 7#include <cpu/x86/msr.h> 8#include <cpu/x86/mtrr.h> 9#include <cpu/x86/post_code.h> 10#include <intelblocks/msr.h> 11#include <intelblocks/post_codes.h> 12#include <cpu/x86/64bit/entry64.inc> 13 14.section .init, "ax", @progbits 15 16.code32 17 18/* 19 * macro: find_free_mtrr 20 * Clobbers: %eax, %ebx, %ecx, %edx. 21 * Returns: 22 * %ebx contains the number of freely available MTRR's. 23 * It should be checked against 0. 24 * %ecx holds the MTRR_BASE of the free MTRR. 25 */ 26.macro find_free_mtrr 27 /* Figure out how many MTRRs we have */ 28 mov $MTRR_CAP_MSR, %ecx 29 rdmsr 30 movzb %al, %ebx /* Number of variable MTRRs */ 31 32 /* Find a free variable MTRR */ 33 movl $MTRR_PHYS_MASK(0), %ecx 341: 35 rdmsr 36 test $MTRR_PHYS_MASK_VALID, %eax 37 jz 2f 38 addl $2, %ecx 39 dec %ebx 40 jnz 1b 412: 42 /* %ecx needs to hold the MTRR_BASE */ 43 decl %ecx 44.endm 45 46/* 47 * macro: clear_car 48 * Clears the region between CONFIG_DCACHE_RAM_BASE and 49 * CONFIG_DCACHE_RAM_BASE + CONFIG_DCACHE_RAM_SIZE to populate 50 * cachelines. 51 * Clobbers %eax, %ecx, %edi. 52 */ 53.macro clear_car 54 /* Clear the cache memory region. This will also fill up the cache */ 55 movl $CONFIG_DCACHE_RAM_BASE, %edi 56 movl $CONFIG_DCACHE_RAM_SIZE, %ecx 57 shr $0x02, %ecx 58 xor %eax, %eax 59 cld 60 rep stosl 61.endm 62 63/* 64 * macro: is_bootguard_nem 65 * Checks if the Bootguard ACM has enabled non eviction mode 66 * Clobbers %eax, %ecx, %edx 67 * Returns %eax and sets/unsets zero flag 68 */ 69.macro is_bootguard_nem 70#if CONFIG(SOC_INTEL_NO_BOOTGUARD_MSR) 71 xorl %eax, %eax 72#else 73 movl $MSR_BOOT_GUARD_SACM_INFO, %ecx 74 rdmsr 75 andl $B_BOOT_GUARD_SACM_INFO_NEM_ENABLED, %eax 76#endif 77.endm 78 79.global bootblock_pre_c_entry 80bootblock_pre_c_entry: 81 82 post_code(POSTCODE_BOOTBLOCK_PRE_C_ENTRY) 83 84/* Bootguard sets up its own CAR and needs separate handling */ 85check_boot_guard: 86 is_bootguard_nem 87 jz no_bootguard 88 89 /* Disable PBE timer */ 90 movl $MSR_BC_PBEC, %ecx 91 movl $B_STOP_PBET, %eax 92 xorl %edx, %edx 93 wrmsr 94 95 jmp setup_car_mtrr 96 97no_bootguard: 98 movl $no_reset, %esp /* return address */ 99 jmp check_mtrr /* Check if CPU properly reset */ 100 101no_reset: 102 post_code(POSTCODE_SOC_NO_RESET) 103 104 /* Clear/disable fixed MTRRs */ 105 mov $fixed_mtrr_list, %ebx 106 xor %eax, %eax 107 xor %edx, %edx 108 109clear_fixed_mtrr: 110 movzwl (%ebx), %ecx 111 wrmsr 112 add $2, %ebx 113 cmp $fixed_mtrr_list_end, %ebx 114 jl clear_fixed_mtrr 115 116 post_code(POSTCODE_SOC_CLEAR_FIXED_MTRRS) 117 118 /* Figure out how many MTRRs we have, and clear them out */ 119 mov $MTRR_CAP_MSR, %ecx 120 rdmsr 121 movzb %al, %ebx /* Number of variable MTRRs */ 122 mov $MTRR_PHYS_BASE(0), %ecx 123 xor %eax, %eax 124 xor %edx, %edx 125 126clear_var_mtrr: 127 wrmsr 128 inc %ecx 129 wrmsr 130 inc %ecx 131 dec %ebx 132 jnz clear_var_mtrr 133 134 post_code(POSTCODE_SOC_CLEAR_VAR_MTRRS) 135 136 /* Configure default memory type to uncacheable (UC) */ 137 mov $MTRR_DEF_TYPE_MSR, %ecx 138 rdmsr 139 /* Clear enable bits and set default type to UC. */ 140 and $~(MTRR_DEF_TYPE_MASK | MTRR_DEF_TYPE_EN | \ 141 MTRR_DEF_TYPE_FIX_EN), %eax 142 wrmsr 143 144setup_car_mtrr: 145 /* Configure MTRR_PHYS_MASK_HIGH for proper addressing above 4GB 146 * based on the physical address size supported for this processor 147 * This is based on read from CPUID EAX = 080000008h, EAX bits [7:0] 148 * 149 * Examples: 150 * MTRR_PHYS_MASK_HIGH = 00000000Fh For 36 bit addressing 151 * MTRR_PHYS_MASK_HIGH = 0000000FFh For 40 bit addressing 152 */ 153 154 movl $0x80000008, %eax /* Address sizes leaf */ 155 cpuid 156 sub $32, %al 157 movzx %al, %eax 158 xorl %esi, %esi 159 bts %eax, %esi 160 dec %esi /* esi <- MTRR_PHYS_MASK_HIGH */ 161 162 post_code(POSTCODE_SOC_SET_UP_CAR_MTRRS) 163 164#if ((CONFIG_DCACHE_RAM_SIZE & (CONFIG_DCACHE_RAM_SIZE - 1)) == 0) 165 find_free_mtrr 166 test %ebx, %ebx 167 jz .halt_forever 168 169 /* Configure CAR region as write-back (WB) */ 170 mov $CONFIG_DCACHE_RAM_BASE, %eax 171 or $MTRR_TYPE_WRBACK, %eax 172 xor %edx,%edx 173 wrmsr 174 175 /* Configure the MTRR mask for the size region */ 176 inc %ecx 177 mov $CONFIG_DCACHE_RAM_SIZE, %eax /* size mask */ 178 dec %eax 179 not %eax 180 or $MTRR_PHYS_MASK_VALID, %eax 181 movl %esi, %edx /* edx <- MTRR_PHYS_MASK_HIGH */ 182 wrmsr 183#elif (CONFIG_DCACHE_RAM_SIZE == 768 * KiB) /* 768 KiB */ 184 find_free_mtrr 185 test %ebx, %ebx 186 jz .halt_forever 187 188 /* Configure CAR region as write-back (WB) */ 189 mov $CONFIG_DCACHE_RAM_BASE, %eax 190 or $MTRR_TYPE_WRBACK, %eax 191 xor %edx,%edx 192 wrmsr 193 194 incl %ecx 195 mov $(512 * KiB), %eax /* size mask */ 196 dec %eax 197 not %eax 198 or $MTRR_PHYS_MASK_VALID, %eax 199 movl %esi, %edx /* edx <- MTRR_PHYS_MASK_HIGH */ 200 wrmsr 201 202 find_free_mtrr 203 test %ebx, %ebx 204 jz .halt_forever 2051: 206 mov $(CONFIG_DCACHE_RAM_BASE + 512 * KiB), %eax 207 or $MTRR_TYPE_WRBACK, %eax 208 xor %edx,%edx 209 wrmsr 210 211 incl %ecx 212 mov $(256 * KiB), %eax /* size mask */ 213 dec %eax 214 not %eax 215 or $MTRR_PHYS_MASK_VALID, %eax 216 movl %esi, %edx /* edx <- MTRR_PHYS_MASK_HIGH */ 217 wrmsr 218#else 219#error "DCACHE_RAM_SIZE is not a power of 2 and setup code is missing" 220#endif 221 post_code(POSTCODE_SOC_BOOTGUARD_SETUP) 222 223 is_bootguard_nem 224 jz no_bootguard_car_continue 225 226 /* 227 * With Bootguard some RO caching of the flash is already set up by 228 * the ACM. It looks like in such a setup 'clear_car' will not properly fill 229 * the cachelines. Fill all the CAR cachelines explicitly using sfence. 230 * This assumes 64 bytes cachelines. 231 */ 232 movl $CONFIG_DCACHE_RAM_BASE, %edi 233 movl $CONFIG_DCACHE_RAM_SIZE, %ecx 234 shr $0x06, %ecx 235 xor %eax, %eax 236 2371: 238 movl %eax, (%edi) 239 sfence 240 add $64, %edi 241 loop 1b 242 243 clear_car 244 245 jmp car_init_done 246 247no_bootguard_car_continue: 248 /* Enable variable MTRRs */ 249 mov $MTRR_DEF_TYPE_MSR, %ecx 250 rdmsr 251 or $MTRR_DEF_TYPE_EN, %eax 252 wrmsr 253 254 /* Enable caching */ 255 mov %cr0, %eax 256 and $~(CR0_CD | CR0_NW), %eax 257 invd 258 mov %eax, %cr0 259 260#if CONFIG(INTEL_CAR_NEM) 261 jmp car_nem 262#elif CONFIG(INTEL_CAR_CQOS) 263 jmp car_cqos 264#elif CONFIG(INTEL_CAR_NEM_ENHANCED) 265 jmp car_nem_enhanced 266#else 267 jmp .halt_forever /* In case nothing has selected */ 268#endif 269 270.global car_init_done 271car_init_done: 272 273 post_code(POSTCODE_SOC_CAR_INIT_DONE) 274 275 /* Setup bootblock stack */ 276 mov $_ecar_stack, %esp 277 278 /* Need to align stack to 16 bytes at call instruction. Account for 279 the two pushes below. */ 280 andl $0xfffffff0, %esp 281 282#if ENV_X86_64 283 setup_longmode $PM4LE 284 285 movd %mm2, %rdi 286 shlq $32, %rdi 287 movd %mm1, %rsi 288 or %rsi, %rdi 289 movd %mm0, %rsi 290#else 291 sub $8, %esp 292 293 /* push TSC value to stack */ 294 movd %mm2, %eax 295 pushl %eax /* tsc[63:32] */ 296 movd %mm1, %eax 297 pushl %eax /* tsc[31:0] */ 298#endif 299 300 /* Copy .data section content to Cache-As-Ram */ 301#include <cpu/x86/copy_data_section.inc> 302 303before_carstage: 304 post_code(POSTCODE_SOC_BEFORE_CARSTAGE) 305 306 call bootblock_c_entry 307 /* Never reached */ 308 309.halt_forever: 310 post_code(POSTCODE_DEAD_CODE) 311 hlt 312 jmp .halt_forever 313 314fixed_mtrr_list: 315 .word MTRR_FIX_64K_00000 316 .word MTRR_FIX_16K_80000 317 .word MTRR_FIX_16K_A0000 318 .word MTRR_FIX_4K_C0000 319 .word MTRR_FIX_4K_C8000 320 .word MTRR_FIX_4K_D0000 321 .word MTRR_FIX_4K_D8000 322 .word MTRR_FIX_4K_E0000 323 .word MTRR_FIX_4K_E8000 324 .word MTRR_FIX_4K_F0000 325 .word MTRR_FIX_4K_F8000 326fixed_mtrr_list_end: 327 328#if CONFIG(INTEL_CAR_NEM) 329.global car_nem 330car_nem: 331 /* Disable cache eviction (setup stage) */ 332 mov $MSR_EVICT_CTL, %ecx 333 rdmsr 334 or $0x1, %eax 335 wrmsr 336 337 post_code(POSTCODE_SOC_CLEARING_CAR) 338 339 clear_car 340 341 post_code(POSTCODE_SOC_DISABLE_CACHE_EVICT) 342 343 /* Disable cache eviction (run stage) */ 344 mov $MSR_EVICT_CTL, %ecx 345 rdmsr 346 or $0x2, %eax 347 wrmsr 348 349 jmp car_init_done 350 351#elif CONFIG(INTEL_CAR_CQOS) 352.global car_cqos 353car_cqos: 354 /* 355 * Create CBM_LEN_MASK based on CBM_LEN 356 * Get CPUID.(EAX=10H, ECX=2H):EAX.CBM_LEN[bits 4:0] 357 */ 358 mov $0x10, %eax 359 mov $0x2, %ecx 360 cpuid 361 and $0x1F, %eax 362 add $1, %al 363 364 mov $1, %ebx 365 mov %al, %cl 366 shl %cl, %ebx 367 sub $1, %ebx 368 369 /* Store the CBM_LEN_MASK in mm3 for later use. */ 370 movd %ebx, %mm3 371 372 /* 373 * Disable both L1 and L2 prefetcher. For yet-to-understood reason, 374 * prefetchers slow down filling cache with rep stos in CQOS mode. 375 */ 376 mov $MSR_PREFETCH_CTL, %ecx 377 rdmsr 378 or $(PREFETCH_L1_DISABLE | PREFETCH_L2_DISABLE), %eax 379 wrmsr 380 381#if (CONFIG_DCACHE_RAM_SIZE == CONFIG_L2_CACHE_SIZE) 382/* 383 * If CAR size is set to full L2 size, mask is calculated as all-zeros. 384 * This is not supported by the CPU/uCode. 385 */ 386#error "CQOS CAR may not use whole L2 cache area" 387#endif 388 389 /* Calculate how many bits to be used for CAR */ 390 xor %edx, %edx 391 mov $CONFIG_DCACHE_RAM_SIZE, %eax /* dividend */ 392 mov $CONFIG_CACHE_QOS_SIZE_PER_BIT, %ecx /* divisor */ 393 div %ecx /* result is in eax */ 394 mov %eax, %ecx /* save to ecx */ 395 mov $1, %ebx 396 shl %cl, %ebx 397 sub $1, %ebx /* resulting mask is in ebx */ 398 399 /* Set this mask for initial cache fill */ 400 mov $MSR_L2_QOS_MASK(0), %ecx 401 rdmsr 402 mov %ebx, %eax 403 wrmsr 404 405 /* Set CLOS selector to 0 */ 406 mov $IA32_PQR_ASSOC, %ecx 407 rdmsr 408 and $~IA32_PQR_ASSOC_MASK, %edx /* select mask 0 */ 409 wrmsr 410 411 /* We will need to block CAR region from evicts */ 412 mov $MSR_L2_QOS_MASK(1), %ecx 413 rdmsr 414 /* Invert bits that are to be used for cache */ 415 mov %ebx, %eax 416 xor $~0, %eax /* invert 32 bits */ 417 418 /* 419 * Use CBM_LEN_MASK stored in mm3 to set bits based on Capacity Bit 420 * Mask Length. 421 */ 422 movd %mm3, %ebx 423 and %ebx, %eax 424 wrmsr 425 426 post_code(POSTCODE_SOC_CLEARING_CAR) 427 428 clear_car 429 430 post_code(POSTCODE_SOC_DISABLE_CACHE_EVICT) 431 432 /* Cache is populated. Use mask 1 that will block evicts */ 433 mov $IA32_PQR_ASSOC, %ecx 434 rdmsr 435 and $~IA32_PQR_ASSOC_MASK, %edx /* clear index bits first */ 436 or $1, %edx /* select mask 1 */ 437 wrmsr 438 439 /* Enable prefetchers */ 440 mov $MSR_PREFETCH_CTL, %ecx 441 rdmsr 442 and $~(PREFETCH_L1_DISABLE | PREFETCH_L2_DISABLE), %eax 443 wrmsr 444 445 jmp car_init_done 446 447#elif CONFIG(INTEL_CAR_NEM_ENHANCED) 448.global car_nem_enhanced 449car_nem_enhanced: 450 /* Disable cache eviction (setup stage) */ 451 mov $MSR_EVICT_CTL, %ecx 452 rdmsr 453 or $0x1, %eax 454 wrmsr 455 post_code(POSTCODE_SOC_CAR_NEM_ENHANCED) 456 457 /* Create n-way set associativity of cache */ 458 xorl %edi, %edi 459find_llc_subleaf: 460 movl %edi, %ecx 461 movl $0x04, %eax 462 cpuid 463 inc %edi 464 and $0xe0, %al /* EAX[7:5] = Cache Level */ 465 cmp $0x60, %al /* Check to see if it is LLC */ 466 jnz find_llc_subleaf 467 468 /* 469 * Calculate the total LLC size 470 * (Line_Size + 1) * (Sets + 1) * (Partitions + 1) * (Ways + 1) 471 * (EBX[11:0] + 1) * (ECX + 1) * (EBX[21:12] + 1) * EBX[31:22] + 1) 472 */ 473 474 mov %ebx, %eax 475 and $0xFFF, %eax 476 inc %eax 477 inc %ecx 478 mul %ecx 479 mov %eax, %ecx 480 mov %ebx, %eax 481 shr $12, %eax 482 and $0x3FF, %eax 483 inc %eax 484 mul %ecx 485 shr $22, %ebx 486 inc %ebx 487 mov %ebx, %edx 488 mul %ebx /* eax now holds total LLC size */ 489 490 /* 491 * The number of the ways that we want to protect from eviction 492 * can be calculated as RW data stack size / way size where way 493 * size is Total LLC size / Total number of LLC ways. 494 */ 495 div %ebx /* way size */ 496 mov %eax, %ecx 497 498 /* 499 * Check if way size if bigger than the cache ram size. 500 * Then we need to allocate just one way for non-eviction 501 * of RW data. 502 */ 503 movl $0x01, %eax 504 mov %eax, %edx /* back up data_ways in edx */ 505 cmp $CONFIG_DCACHE_RAM_SIZE, %ecx 506 jnc set_eviction_mask 507 508 /* 509 * RW data size / way size is equal to number of 510 * ways to be configured for non-eviction 511 */ 512 mov $CONFIG_DCACHE_RAM_SIZE, %eax 513 div %ecx 514 mov %eax, %edx /* back up data_ways in edx */ 515 mov %eax, %ecx 516 movl $0x01, %eax 517 shl %cl, %eax 518 subl $0x01, %eax 519 520set_eviction_mask: 521 mov %ebx, %edi /* back up number of ways */ 522 mov %eax, %esi /* back up the non-eviction mask */ 523#if CONFIG(CAR_HAS_SF_MASKS) 524 mov %edx, %eax /* restore data_ways in eax */ 525 /* 526 * Calculate SF masks 2: 527 * 1. if CONFIG_SF_MASK_2WAYS_PER_BIT: data_ways = data_ways / 2 528 * 2. Program MSR 0x1892 Non-Eviction Mask #2 529 * IA32_CR_SF_QOS_MASK_2 = ((1 << data_ways) - 1) 530 */ 531#if CONFIG(SF_MASK_2WAYS_PER_BIT) 532 cmp $0x01, %eax /* Skip Step 1 if data_ways = 1 */ 533 jz program_sf2 534 movl $0x01, %ecx /* Step 1 */ 535 shr %cl, %eax 536#endif 537 /* Step 2 */ 538 mov %eax, %ecx 539 movl $0x01, %eax 540 shl %cl, %eax 541 subl $0x01, %eax 542program_sf2: 543 mov %eax, %ebx /* back up IA32_CR_SF_QOS_MASK_2 in ebx */ 544 xorl %edx, %edx 545 mov $IA32_CR_SF_QOS_MASK_2, %ecx 546 wrmsr 547 548 /* 549 * Calculate the SF Mask 1: 550 * 1. Calculate SFWayCnt = IA32_SF_QOS_INFO & Bit [5:0] 551 * 2. if CONFIG_SF_MASK_2WAYS_PER_BIT: SFWayCnt = SFWayCnt / 2 552 * 3. Set SF_MASK_1 = ((1 << SFWayCnt) - 1) - IA32_CR_SF_QOS_MASK_2 553 */ 554 mov $IA32_SF_QOS_INFO, %ecx 555 rdmsr 556 and $IA32_SF_WAY_COUNT_MASK, %eax /* Step 1 */ 557#if CONFIG(SF_MASK_2WAYS_PER_BIT) 558 /* Assumption: skip checking SFWayCnt = 1 i.e. 1 way LLC (which is not practical) */ 559 movl $0x01, %ecx /* Step 2 */ 560 shr %cl, %eax 561#endif 562 /* Step 3 */ 563 mov %eax, %ecx 564 movl $0x01, %eax 565 shl %cl, %eax 566 subl $0x01, %eax 567 sub %ebx, %eax 568 xorl %edx, %edx 569 movl $IA32_CR_SF_QOS_MASK_1, %ecx 570 wrmsr 571#endif 572#if CONFIG(CAR_HAS_L3_PROTECTED_WAYS) 573 /* Set MSR 0xC85 L3_Protected_ways = ((1 << data ways) - 1) */ 574 mov %esi, %eax 575 xorl %edx, %edx 576 mov $IA32_L3_PROTECTED_WAYS, %ecx 577 wrmsr 578#endif 579 /* 580 * Program MSR 0xC91 IA32_L3_MASK_1 581 * This MSR contain one bit per each way of LLC 582 * - If this bit is '0' - the way is protected from eviction 583 * - If this bit is '1' - the way is not protected from eviction 584 */ 585 mov $0x1, %eax 586 mov %edi, %ecx 587 shl %cl, %eax 588 subl $0x01, %eax 589 mov %eax, %ecx 590 mov %esi, %eax 591 592 xor $~0, %eax /* invert 32 bits */ 593 and %ecx, %eax 594 movl $IA32_L3_MASK_1, %ecx 595 xorl %edx, %edx 596 wrmsr 597 /* 598 * Program MSR 0xC92 IA32_L3_MASK_2 599 * This MSR contain one bit per each way of LLC 600 * - If this bit is '0' - the way is protected from eviction 601 * - If this bit is '1' - the way is not protected from eviction 602 */ 603 mov %esi, %eax 604 movl $IA32_L3_MASK_2, %ecx 605 xorl %edx, %edx 606 wrmsr 607 /* 608 * Set IA32_PQR_ASSOC 609 * 610 * Possible values: 611 * 0: Default value, no way mask should be applied 612 * 1: Apply way mask 1 to LLC 613 * 2: Apply way mask 2 to LLC 614 * 3: Shouldn't be use in NEM Mode 615 */ 616 movl $IA32_PQR_ASSOC, %ecx 617 xorl %eax, %eax 618 xorl %edx, %edx 619#if CONFIG(COS_MAPPED_TO_MSB) 620 movl $0x02, %edx 621#else 622 movl $0x02, %eax 623#endif 624 wrmsr 625 626 clear_car 627 628 /* 629 * Set IA32_PQR_ASSOC 630 * At this stage we apply LLC_WAY_MASK_1 to the cache. 631 */ 632 movl $IA32_PQR_ASSOC, %ecx 633 xorl %eax, %eax 634 xorl %edx, %edx 635#if CONFIG(COS_MAPPED_TO_MSB) 636 movl $0x01, %edx 637#else 638 movl $0x01, %eax 639#endif 640 wrmsr 641 642 post_code(POSTCODE_SOC_DISABLE_CACHE_EVICT) 643 /* 644 * Enable No-Eviction Mode Run State by setting 645 * NO_EVICT_MODE MSR 2E0h bit [1] = '1'. 646 */ 647 648 movl $MSR_EVICT_CTL, %ecx 649 rdmsr 650 orl $0x02, %eax 651 wrmsr 652 653 jmp car_init_done 654#endif 655