1 /*============================================================================== 2 Copyright(c) 2017 Intel Corporation 3 4 Permission is hereby granted, free of charge, to any person obtaining a 5 copy of this software and associated documentation files(the "Software"), 6 to deal in the Software without restriction, including without limitation 7 the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 and / or sell copies of the Software, and to permit persons to whom the 9 Software is furnished to do so, subject to the following conditions: 10 11 The above copyright notice and this permission notice shall be included 12 in all copies or substantial portions of the Software. 13 14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 15 OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 OTHER DEALINGS IN THE SOFTWARE. 21 ============================================================================*/ 22 #pragma once 23 24 #ifdef __cplusplus 25 extern "C" { 26 #endif 27 #include "GmmCachePolicyExt.h" 28 29 #define ALWAYS_OVERRIDE 0xffffffff 30 #define NEVER_OVERRIDE 0x0 31 32 // IA PAT MSR Register 33 #define GMM_IA32e_PAT_MSR 0x277 34 35 #define GMM_PAT_ERROR ((uint32_t)-1) 36 37 typedef struct GMM_CACHE_POLICY_ELEMENT_REC 38 { 39 uint32_t IDCode; 40 union { 41 struct{ 42 uint64_t LLC : 1; 43 uint64_t ELLC : 1; 44 uint64_t L3 : 1; 45 uint64_t WT : 1; 46 uint64_t AGE : 2; 47 uint64_t AOM : 1; 48 uint64_t LeCC_SCC : 3; 49 uint64_t L3_SCC : 3; 50 uint64_t SCF : 1; // Snoop Control Field for BXT 51 uint64_t CoS : 2; // Class of Service, driver default to Class 0 52 uint64_t SSO : 2; // Self Snoop Override control and value 53 uint64_t HDCL1 : 1; // HDC L1 caching enable/disable 54 uint64_t L3Eviction : 2; // Specify L3-eviction type (NA, ReadOnly, Standard, Special) 55 uint64_t SegOv : 3; // Override seg-pref (none, local-only, sys-only, etc) 56 uint64_t GlbGo : 1; // Global GO point - L3 or Memory 57 uint64_t UcLookup : 1; // Snoop L3 for uncached 58 uint64_t L1CC : 3; // L1 Cache Control 59 uint64_t Initialized : 1; 60 uint64_t L2CC : 2; // media internal cache 0:UC, 1:WB 61 uint64_t L4CC : 2; // L4 memory cache 0: UC, 1:WB, 2: WT 62 uint64_t Coherency : 2; // 0 non-coh, 1: 1 way coh IA snoop 2: 2 way coh IA GPU snopp 63 uint64_t CoherentPATIndex : 5; 64 uint64_t CoherentPATIndexHigherBit : 1; // From Xe2 onwards it requires 6 bit to represent PATIndex. Hence using this single bit (MSB) as extension of the above field CoherentPATIndex:5 65 uint64_t PATIndexCompressed : 6; 66 uint64_t L3CC : 2; // 0:UC, 1:WB 2:WB_T_Display, 3:WB_T_App 67 uint64_t L3CLOS : 2; // Class of service 68 uint64_t IgnorePAT : 1; // Ignore PAT 1 = Override by MOCS, 0 = Defer to PAT 69 uint64_t Reserved : 11; 70 }; 71 uint64_t Value; 72 }; 73 74 MEMORY_OBJECT_CONTROL_STATE MemoryObjectOverride; 75 76 union 77 { 78 MEMORY_OBJECT_CONTROL_STATE MemoryObjectNoOverride; 79 uint32_t PATIndex; 80 }; 81 82 GMM_PTE_CACHE_CONTROL_BITS PTE; 83 uint32_t Override; 84 uint32_t IsOverridenByRegkey; // Flag to indicate If usage settings are overridden by regkey 85 }GMM_CACHE_POLICY_ELEMENT; 86 87 // One entry in the SKL/CNL cache lookup table 88 typedef struct GMM_CACHE_POLICY_TBL_ELEMENT_REC { 89 union { 90 struct 91 { 92 uint32_t Cacheability : 2; // LLC/eDRAM Cacheability Control (LeCC). 93 uint32_t TargetCache : 2; // Target Cache (TC). 94 uint32_t LRUM : 2; // LRU (Cache Replacement) Management (LRUM). 95 uint32_t AOM : 1; // Do Not Allocate on Miss (AOM) - Not used 96 uint32_t ESC : 1; // Enable Skip Caching (ESC) on LLC 97 uint32_t SCC : 3; // Skip Caching Control (SCC). 98 uint32_t PFM : 3; // Page Fault Mode (PFM) 99 uint32_t SCF : 1; // Snoop Control Field (SCF) 100 uint32_t CoS : 2; // Class of Service (COS)- CNL+ 101 uint32_t SelfSnoop : 2; // Self Snoop override or not MIDI settings - CNL+ 102 uint32_t Reserved : 13; 103 } ; 104 105 uint32_t DwordValue; 106 107 union { 108 struct 109 { 110 uint32_t Reserved0 : 2; 111 uint32_t L4CC : 2; 112 uint32_t Reserved1 : 2; 113 uint32_t Reserved2 : 2; 114 uint32_t igPAT : 1; // selection between MOCS and PAT 115 uint32_t Reserved3 : 23; 116 }; 117 118 uint32_t DwordValue; 119 120 } Xe_LPG; 121 122 } LeCC; 123 124 union { 125 struct 126 { 127 uint16_t ESC : 1; // Enable Skip Caching (ESC) for L3. 128 uint16_t SCC : 3; // Skip Caching Control (SCC) for L3. 129 uint16_t Cacheability : 2; // L3 Cacheability Control (L3CC). 130 uint16_t GlobalGo : 1; // Global Go (GLBGO). 131 uint16_t UCLookup : 1; // UC L3 Lookup (UcL3Lookup). 132 uint16_t Reserved : 8; 133 } ; 134 uint16_t UshortValue; 135 136 union 137 { 138 struct 139 { 140 uint16_t Reserved : 2; 141 uint16_t L4CC : 2; 142 uint16_t L3CC : 2; 143 uint16_t L3CLOS : 2; 144 uint16_t igPAT : 1; // selection between MOCS and PAT 145 uint16_t Reserved0: 7; 146 }; 147 } PhysicalL3; 148 } L3; 149 150 uint8_t HDCL1; 151 152 } GMM_CACHE_POLICY_TBL_ELEMENT; 153 154 typedef enum GMM_IA32e_MEMORY_TYPE_REC 155 { 156 GMM_IA32e_UC = 0x0, 157 GMM_IA32e_WC = 0x1, 158 GMM_IA32e_WT = 0x4, 159 GMM_IA32e_WP = 0x5, 160 GMM_IA32e_WB = 0x6, 161 GMM_IA32e_UCMINUS = 0x7 162 } GMM_IA32e_MEMORY_TYPE; 163 164 typedef enum GMM_GFX_MEMORY_TYPE_REC 165 { 166 GMM_GFX_UC_WITH_FENCE = 0x0, 167 GMM_GFX_WC = 0x1, 168 GMM_GFX_WT = 0x2, 169 GMM_GFX_WB = 0x3 170 } GMM_GFX_MEMORY_TYPE; 171 172 typedef enum GMM_L4_CACHING_POLICY_REC 173 { 174 GMM_CP_COHERENT_WB = 0x0, 175 GMM_CP_NON_COHERENT_WB = 0x0, 176 GMM_CP_NON_COHERENT_WT = 0x1, 177 GMM_CP_NON_COHERENT_UC = 0x3, 178 } GMM_L4_CACHING_POLICY; 179 180 // This Enums represent the GMM indicative values for L1/L3/L4 cache attributes. 181 typedef enum GMM_CACHING_POLICY_REC 182 { 183 GMM_UC = 0x0, //uncached 184 GMM_WB = 0x1, // Write back 185 GMM_WT = 0x2, // write-through 186 GMM_WBTD = 0x3, // WB_T_Display 187 GMM_WBTA = 0x4, // WB_T_App 188 GMM_WBP = 0x5, // write bypass mode 189 GMM_WS = 0x6, // Write-Streaming 190 } GMM_CACHING_POLICY; 191 192 typedef enum GMM_COHERENCY_TYPE_REC 193 { 194 GMM_NON_COHERENT_NO_SNOOP = 0x0, 195 GMM_COHERENT_ONE_WAY_IA_SNOOP = 0x1, 196 GMM_COHERENT_TWO_WAY_IA_GPU_SNOOP = 0x2 197 } GMM_COHERENCY_TYPE; 198 199 typedef enum GMM_GFX_COHERENCY_TYPE_REC 200 { 201 GMM_GFX_NON_COHERENT_NO_SNOOP = 0x0, 202 GMM_GFX_NON_COHERENT = 0x1, 203 GMM_GFX_COHERENT_ONE_WAY_IA_SNOOP = 0x2, 204 GMM_GFX_COHERENT_TWO_WAY_IA_GPU_SNOOP = 0x3 205 } GMM_GFX_COHERENCY_TYPE; 206 207 typedef enum GMM_GFX_PAT_TYPE_REC 208 { 209 GMM_GFX_PAT_WB_COHERENT = 0x0, // WB + Snoop : Atom 210 GMM_GFX_PAT_WC = 0x1, 211 GMM_GFX_PAT_WB_MOCSLESS = 0x2, // WB + eLLC/LLC : MOCSLess Resource (PageTables) 212 GMM_GFX_PAT_WB = 0x3, 213 GMM_GFX_PAT_WT = 0x4, 214 GMM_GFX_PAT_UC = 0x5 215 } GMM_GFX_PAT_TYPE; 216 217 typedef enum GMM_GFX_PAT_IDX_REC 218 { 219 PAT0 = 0x0, // Will be tied to GMM_GFX_PAT_WB_COHERENT or GMM_GFX_PAT_UC depending on WaGttPat0WB 220 PAT1, // Will be tied to GMM_GFX_PAT_UC or GMM_GFX_PAT_WB_COHERENT depending on WaGttPat0WB 221 PAT2, // Will be tied to GMM_GFX_PAT_WB_MOCSLESS 222 PAT3, // Will be tied to GMM_GFX_PAT_WB 223 PAT4, // Will be tied to GMM_GFX_PAT_WT 224 PAT5, // Will be tied to GMM_GFX_PAT_WC 225 PAT6, // Will be tied to GMM_GFX_PAT_WC 226 PAT7, // Will be tied to GMM_GFX_PAT_WC 227 // Additional registers 228 PAT8, 229 PAT9, 230 PAT10, 231 PAT11, 232 PAT12, 233 PAT13, 234 PAT14, 235 PAT15 236 } GMM_GFX_PAT_IDX; 237 238 #define GFX_IS_ATOM_PLATFORM(pGmmLibContext) (GmmGetSkuTable(pGmmLibContext)->FtrLCIA) 239 240 typedef enum GMM_GFX_TARGET_CACHE_REC 241 { 242 GMM_GFX_TC_ELLC_ONLY = 0x0, 243 GMM_GFX_TC_LLC_ONLY = 0x1, 244 GMM_GFX_TC_ELLC_LLC = 0x3 245 } GMM_GFX_TARGET_CACHE; 246 247 typedef union GMM_PRIVATE_PAT_REC { 248 249 struct 250 { 251 uint32_t MemoryType : 2; 252 uint32_t TargetCache : 2; 253 uint32_t Age : 2; 254 uint32_t Snoop : 1; 255 uint32_t Reserved : 1; 256 }PreGen10; 257 #if (IGFX_GEN >= IGFX_GEN10) 258 struct 259 { 260 uint32_t MemoryType : 2; 261 uint32_t TargetCache : 2; 262 uint32_t Age : 2; 263 uint32_t Reserved1 : 2; 264 uint32_t CoS : 2; 265 uint32_t Reserved2 : 22; 266 }Gen10; 267 #endif 268 struct 269 { 270 uint32_t MemoryType : 2; 271 uint32_t Reserved : 30; 272 }Gen12; 273 274 struct 275 { 276 uint32_t MemoryType : 2; //L3 277 uint32_t L3CLOS : 2; 278 uint32_t Reserved : 28; 279 } Xe_HPC; 280 281 struct 282 { 283 uint32_t Coherency : 2; 284 uint32_t L4CC : 2; 285 uint32_t Reserved : 28; 286 }Xe_LPG; 287 288 struct 289 { 290 uint32_t Coherency : 2; 291 uint32_t L4CC : 2; 292 uint32_t L3CC : 2; 293 uint32_t L3CLOS : 2; 294 uint32_t Reserved1 : 1; 295 uint32_t LosslessCompressionEn: 1; 296 uint32_t NoCachingPromote : 1; 297 uint32_t Reserved2 : 21; 298 } Xe2; 299 300 uint32_t Value; 301 302 } GMM_PRIVATE_PAT; 303 304 #ifdef _WIN32 305 306 #define GMM_CACHE_POLICY_OVERRIDE_REGISTY_PATH_REGISTRY_UMD "SOFTWARE\\Intel\\IGFX\\CachePolicy\\" 307 #define GMM_CACHE_POLICY_OVERRIDE_REGISTY_PATH_REGISTRY_KMD "\\REGISTRY\\MACHINE\\" GMM_CACHE_POLICY_OVERRIDE_REGISTY_PATH_REGISTRY_UMD 308 309 // No local variables or ternary (?:) operators should be defined in the 310 // REG_OVERRIDE macro block. This block will generate hundreds of instances 311 // of itself in the OverrideCachePolicy, each of which will geneate a 312 // copy of these local variables on the stack. The ternary operator will 313 // also generate local temparoary stack variables, and all these add up and 314 // can cause a stack overflow. 315 316 #define READOVERRIDES(Usage) \ 317 if (!REGISTRY_OVERRIDE_READ(Usage,LLC)) LLC = -1; \ 318 if (!REGISTRY_OVERRIDE_READ(Usage,ELLC)) ELLC = -1; \ 319 if (!REGISTRY_OVERRIDE_READ(Usage,L3)) L3 = -1; \ 320 if (!REGISTRY_OVERRIDE_READ(Usage,Age)) Age = -1; \ 321 if (!REGISTRY_OVERRIDE_READ(Usage,WT)) WT = -1; \ 322 if (!REGISTRY_OVERRIDE_READ(Usage,AOM)) AOM = -1; \ 323 if (!REGISTRY_OVERRIDE_READ(Usage,LeCC_SCC)) LeCC_SCC = -1; \ 324 if (!REGISTRY_OVERRIDE_READ(Usage,L3_SCC)) L3_SCC = -1; \ 325 if (!REGISTRY_OVERRIDE_READ(Usage,SCF)) SCF = -1; \ 326 if (!REGISTRY_OVERRIDE_READ(Usage,SSO)) SSO = -1; \ 327 if (!REGISTRY_OVERRIDE_READ(Usage,CoS)) CoS = -1; \ 328 if (!REGISTRY_OVERRIDE_READ(Usage,HDCL1)) HDCL1 = -1; \ 329 if (!REGISTRY_OVERRIDE_READ(Usage,L3Eviction)) L3Eviction = -1; \ 330 if (!REGISTRY_OVERRIDE_READ(Usage,GlbGo)) GlbGo = -1; \ 331 if (!REGISTRY_OVERRIDE_READ(Usage,UcLookup)) UcLookup = -1; \ 332 333 334 335 #define SETOVERRIDES(Usage) \ 336 if (LLC != -1) \ 337 { \ 338 pCachePolicy[Usage].LLC = LLC; \ 339 } \ 340 if (ELLC != -1) \ 341 { \ 342 pCachePolicy[Usage].ELLC = ELLC; \ 343 } \ 344 if (L3 != -1) \ 345 { \ 346 pCachePolicy[Usage].L3 = L3; \ 347 } \ 348 if (WT != -1) \ 349 { \ 350 pCachePolicy[Usage].WT = WT; \ 351 } \ 352 if (Age != -1) \ 353 { \ 354 pCachePolicy[Usage].AGE = Age; \ 355 } \ 356 if (AOM != -1) \ 357 { \ 358 pCachePolicy[Usage].AOM = AOM; \ 359 } \ 360 if (LeCC_SCC != -1) \ 361 { \ 362 pCachePolicy[Usage].LeCC_SCC = LeCC_SCC; \ 363 } \ 364 if (L3_SCC != -1) \ 365 { \ 366 pCachePolicy[Usage].L3_SCC = L3_SCC; \ 367 } \ 368 if (SCF != -1) \ 369 { \ 370 pCachePolicy[Usage].SCF = SCF; \ 371 } \ 372 if (SSO != -1) \ 373 { \ 374 pCachePolicy[Usage].SSO = SSO; \ 375 } \ 376 if (CoS != -1) \ 377 { \ 378 pCachePolicy[Usage].CoS = CoS; \ 379 } \ 380 if (HDCL1 != -1) \ 381 { \ 382 pCachePolicy[Usage].HDCL1 = HDCL1; \ 383 } \ 384 if (L3Eviction != -1) \ 385 { \ 386 pCachePolicy[Usage].L3Eviction = L3Eviction; \ 387 } \ 388 if (GlbGo != -1) \ 389 { \ 390 pCachePolicy[Usage].GlbGo = GlbGo; \ 391 } \ 392 if (UcLookup != -1) \ 393 { \ 394 pCachePolicy[Usage].UcLookup = UcLookup; \ 395 } \ 396 { \ 397 pCachePolicy[Usage].IsOverridenByRegkey = 1; \ 398 } 399 400 #ifdef __GMM_KMD__ 401 #define REG_OVERRIDE(Usage) \ 402 { \ 403 enum {IGNORED = 0, CURRENT, DEFAULT, UNCACHED}; \ 404 \ 405 REGISTRY_OVERRIDE_READ(Usage,Enable); \ 406 \ 407 if (Enable && (GenerateKeys != UNCACHED)) \ 408 { \ 409 READOVERRIDES(Usage) \ 410 } \ 411 \ 412 if (GenerateKeys == DEFAULT || (GenerateKeys == CURRENT && !Enable)) \ 413 { \ 414 REGISTRY_OVERRIDE_WRITE(Usage,LLC, pCachePolicy[Usage].LLC); \ 415 REGISTRY_OVERRIDE_WRITE(Usage,ELLC,pCachePolicy[Usage].ELLC); \ 416 REGISTRY_OVERRIDE_WRITE(Usage,L3, pCachePolicy[Usage].L3); \ 417 REGISTRY_OVERRIDE_WRITE(Usage,WT, pCachePolicy[Usage].WT); \ 418 REGISTRY_OVERRIDE_WRITE(Usage,Age, pCachePolicy[Usage].AGE); \ 419 REGISTRY_OVERRIDE_WRITE(Usage,AOM, pCachePolicy[Usage].AOM); \ 420 REGISTRY_OVERRIDE_WRITE(Usage,LeCC_SCC, pCachePolicy[Usage].LeCC_SCC); \ 421 REGISTRY_OVERRIDE_WRITE(Usage,L3_SCC, pCachePolicy[Usage].L3_SCC); \ 422 REGISTRY_OVERRIDE_WRITE(Usage,SCF, pCachePolicy[Usage].SCF); \ 423 REGISTRY_OVERRIDE_WRITE(Usage,SSO, pCachePolicy[Usage].SSO); \ 424 REGISTRY_OVERRIDE_WRITE(Usage,CoS, pCachePolicy[Usage].CoS); \ 425 REGISTRY_OVERRIDE_WRITE(Usage,HDCL1, pCachePolicy[Usage].HDCL1); \ 426 REGISTRY_OVERRIDE_WRITE(Usage,L3Eviction, pCachePolicy[Usage].L3Eviction); \ 427 REGISTRY_OVERRIDE_WRITE(Usage,Enable,0); \ 428 REGISTRY_OVERRIDE_WRITE(Usage,GlbGo, pCachePolicy[Usage].GlbGo); \ 429 REGISTRY_OVERRIDE_WRITE(Usage,UcLookup, pCachePolicy[Usage].UcLookup); \ 430 431 } \ 432 else if (GenerateKeys == UNCACHED || GenerateKeys == CURRENT) \ 433 { \ 434 if (GenerateKeys == UNCACHED) \ 435 { \ 436 Enable = 1; \ 437 } \ 438 REGISTRY_OVERRIDE_WRITE(Usage,LLC, LLC); \ 439 REGISTRY_OVERRIDE_WRITE(Usage,ELLC,ELLC); \ 440 REGISTRY_OVERRIDE_WRITE(Usage,L3, L3); \ 441 REGISTRY_OVERRIDE_WRITE(Usage,WT, WT); \ 442 REGISTRY_OVERRIDE_WRITE(Usage,Age, Age); \ 443 REGISTRY_OVERRIDE_WRITE(Usage,AOM, AOM); \ 444 REGISTRY_OVERRIDE_WRITE(Usage,LeCC_SCC, LeCC_SCC); \ 445 REGISTRY_OVERRIDE_WRITE(Usage,L3_SCC, L3_SCC); \ 446 REGISTRY_OVERRIDE_WRITE(Usage,SCF, SCF); \ 447 REGISTRY_OVERRIDE_WRITE(Usage,SSO, SSO); \ 448 REGISTRY_OVERRIDE_WRITE(Usage,CoS, CoS); \ 449 REGISTRY_OVERRIDE_WRITE(Usage,HDCL1, HDCL1); \ 450 REGISTRY_OVERRIDE_WRITE(Usage,L3Eviction, L3Eviction); \ 451 REGISTRY_OVERRIDE_WRITE(Usage,Enable,Enable); \ 452 REGISTRY_OVERRIDE_WRITE(Usage,GlbGo, GlbGo); \ 453 REGISTRY_OVERRIDE_WRITE(Usage,UcLookup, UcLookup); \ 454 } \ 455 \ 456 if (Enable) \ 457 { \ 458 SETOVERRIDES(Usage) \ 459 } \ 460 \ 461 UsageCount++; \ 462 } 463 #else 464 #define REG_OVERRIDE(Usage) \ 465 { \ 466 REGISTRY_OVERRIDE_READ(Usage,Enable); \ 467 if (Enable) \ 468 { \ 469 READOVERRIDES(Usage) \ 470 SETOVERRIDES(Usage) \ 471 } \ 472 UsageCount++; \ 473 } 474 #endif 475 476 #endif // #if _WIN32 477 478 #if __cplusplus 479 #include "GmmCachePolicyCommon.h" 480 #include "CachePolicy/GmmCachePolicyGen8.h" 481 #include "CachePolicy/GmmCachePolicyGen9.h" 482 #endif 483 484 // Function Prototypes 485 GMM_STATUS GmmInitializeCachePolicy(); 486 GMM_GFX_MEMORY_TYPE GmmGetWantedMemoryType(void *pLibContext, GMM_CACHE_POLICY_ELEMENT CachePolicy); 487 488 // Used for GMM ULT testing. 489 #ifdef __GMM_ULT 490 extern void __GmmULTCachePolicy(GMM_CACHE_POLICY_ELEMENT *pCachePolicy); 491 #endif 492 #ifdef __cplusplus 493 } 494 #endif 495