1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * Copyright (C) 2020 SUSE LLC 4 * Author: Nicolai Stange <[email protected]> 5 * LTP port: Martin Doucha <[email protected]> 6 */ 7 8.set KVM_TCONF, 32 9.set KVM_TEXIT, 0xff 10.set RESULT_ADDRESS, 0xfffff000 11.set KVM_GDT_SIZE, 32 12 13.set MSR_VM_HSAVE_PA, 0xc0010117 14 15/* 16 * This section will be allocated at address 0x1000 and 17 * jumped to from the reset stub provided by kvm_run. 18 */ 19.code16 20.section .init.protected_mode, "ax" 21real_mode_entry: 22 cli 23 24 lgdt kvm_gdt32_desc 25 26 mov $0x11, %eax 27 mov %eax, %cr0 28 29 jmp $3 * 8, $protected_mode_entry 30 31.code32 32protected_mode_entry: 33 mov $2 * 8, %eax 34 mov %eax, %ds 35 mov %eax, %es 36 jmp init_memlayout 37 38.section .init.gdt32, "a", @progbits 39 40.macro gdt32_entry type:req l=0 d=0 dpl=0 limit=0xfffff g=1 p=1 41 .4byte \limit & 0xffff 42 .2byte (\type << 8) | (\dpl << 13) | (\p << 15) 43 .2byte (\limit >> 16) | (\l << 5) | (\d << 6) | (\g << 7) 44.endm 45.align 8 46kvm_gdt32: 47 .8byte 0 48 gdt32_entry type=0x1a l=1 /* Code segment long mode */ 49 gdt32_entry type=0x12 /* Data segment, writable */ 50 gdt32_entry type=0x1a l=0 d=1 /* Code segment protected_mode, 32bits */ 51 52.Lgdt32_end: 53kvm_gdt32_desc: 54 .2byte .Lgdt32_end - kvm_gdt32 - 1 55 .4byte kvm_gdt32 56 57.section .data.strings, "aS", @progbits 58source_filename: 59 .ascii "bootstrap_x86_64.S\0" 60 61long_mode_err: 62 .ascii "Virtual CPU does not support 64bit mode\0" 63 64.code32 65.section .init.memlayout, "ax" 66init_memlayout: 67 /* 68 * Identity-map the first 2GB of virtual address space. 69 */ 70 lea kvm_pagetable, %edi 71 72 /* 73 * Set the first entry of kvm_pagetable (level 1) and fill the rest 74 * of the page with zeroes. 75 */ 76 lea kvm_pgtable_l2, %esi 77 movl %esi, %ebx 78 orl $0x3, %ebx /* Flags: present, writable */ 79 movl %ebx, (%edi) 80 addl $4, %edi 81 movl $1023, %ecx 82 xor %eax, %eax 83 rep stosl 84 85 /* 86 * Set the first four entries of kvm_pgtable_l2 and fill the rest 87 * of the page with zeroes. 88 */ 89 mov %esi, %edi 90 lea kvm_pgtable_l3, %esi 91 movl %esi, %eax 92 mov $4, %ecx 93 941: movl %eax, %ebx 95 orl $0x3, %ebx /* Flags: present, writable */ 96 movl %ebx, (%edi) 97 movl $0, 4(%edi) 98 addl $8, %edi 99 addl $4096, %eax 100 dec %ecx 101 jnz 1b 102 103 movl $1016, %ecx 104 xor %eax, %eax 105 rep stosl 106 107 /* Fill kvm_pgtable_l3 with pointers to kvm_pgtable_l4 */ 108 mov %esi, %edi 109 lea kvm_pgtable_l4, %esi 110 movl %esi, %eax 111 mov $4 * 512, %ecx 112 1131: movl %eax, %ebx 114 orl $0x3, %ebx /* Flags: present, writable */ 115 movl %ebx, (%edi) 116 movl $0, 4(%edi) 117 addl $8, %edi 118 addl $4096, %eax 119 dec %ecx 120 jnz 1b 121 122 /* Fill kvm_pgtable_l4 with identity map of the first 2GB. */ 123 movl %esi, %edi 124 movl $2 * 512 * 512, %ecx 125 xor %eax, %eax 126 1271: movl %eax, %ebx 128 orl $0x3, %ebx /* Flags: present, writable */ 129 movl %ebx, (%edi) 130 movl $0, 4(%edi) 131 addl $8, %edi 132 addl $4096, %eax 133 dec %ecx 134 jnz 1b 135 136 /* Mark the upper 2GB as unmapped except for the last page. */ 137 movl $4 * 512 * 512 - 2, %ecx 138 xor %eax, %eax 139 rep stosl 140 movl $0xfffff003, (%edi) 141 movl $0, 4(%edi) 142 143 /* 144 * Now that the identity-map pagestables have been populated, 145 * we're ready to install them at CR3 and switch to long mode. 146 */ 147 /* Enable CR4.PAE */ 148 movl %cr4, %eax 149 btsl $5, %eax 150 movl %eax, %cr4 151 152 lea kvm_pagetable, %eax 153 movl %eax, %cr3 154 155 /* Check if the CPU supports long mode. */ 156 movl $0x80000000, %eax 157 cpuid 158 cmpl $0x80000000, %eax 159 jg 1f 160 movl $KVM_TCONF, %edi 161 lea long_mode_err, %esi 162 jmp init_error 1631: 164 movl $0x80000001, %eax 165 cpuid 166 bt $29, %edx 167 jc 1f 168 movl $KVM_TCONF, %edi 169 lea long_mode_err, %esi 170 jmp init_error 1711: 172 173 /* Activate EFER.LME to enable long mode. */ 174 movl $0xc0000080, %ecx 175 rdmsr 176 btsl $8, %eax 177 wrmsr 178 179 /* Enable CR0.PG and CR0.WP */ 180 movl %cr0, %eax 181 btsl $31, %eax 182 btsl $16, %eax 183 movl %eax, %cr0 184 185 /* Long jmp to load the long mode %cs. */ 186 jmp $1 * 8, $long_mode_entry 187 188init_error: 189 /* Write error info to test result structure and exit VM */ 190 /* Equivalent to tst_brk() but using only 32bit instructions */ 191 movl %edi, RESULT_ADDRESS 192 movl $RESULT_ADDRESS+4, %edi 193 movl $0, (%edi) 194 lea source_filename, %eax 195 movl %eax, 4(%edi) 196 movl $0, 8(%edi) 197 addl $12, %edi 198 xor %edx, %edx 199 2001: movzbl (%esi,%edx,1), %eax 201 movb %al, (%edi,%edx,1) 202 inc %edx 203 test %al, %al 204 jne 1b 205 hlt 206 jmp kvm_exit 207 208.code64 209long_mode_entry: 210 lgdt kvm_gdt_desc 211 212 /* 213 * Reset data segment selectors to NULL selector and 214 * initialize stack. 215 */ 216 xor %eax, %eax 217 mov %eax, %ds 218 mov %eax, %es 219 mov %eax, %ss 220 lea kvm_stack_top, %rsp 221 222 /* 223 * Strictly speaking a TSS should not be required 224 * and experiments confirm that. However, we 225 * might perhaps want to play games with the 226 * interrupt/exception stacks in the future, so 227 * install a minimal one now. 228 */ 229 lea kvm_tss, %rdx 230 movq %rdx, %rdi 231 movq $.Ltss_end - kvm_tss, %rsi 232 call memzero 233 234 movq %rsp, 4(%rdx) 235 236 /* 237 * Create a 16 byte descriptor starting at the 238 * 3rd 8-byte GDT slot.xs 239 */ 240 movq %rdx, %rax 241 shl $40, %rax 242 shr $24, %rax 243 movq %rdx, %rbx 244 shr $24, %rbx 245 shl $56, %rbx 246 or %rbx, %rax 247 movq $0x89, %rbx 248 shl $40, %rbx 249 or $.Ltss_end - kvm_tss - 1, %rbx 250 or %rbx, %rax 251 shr $32, %rdx 252 253 lea kvm_gdt + 2*8, %rdi 254 mov %rax, (%rdi) 255 mov %rdx, 8(%rdi) 256 257 mov $2 * 8, %ax 258 ltr %ax 259 260 261 /* Configure and enable interrupts */ 262 call kvm_init_interrupts 263 lidt kvm_idt_desc 264 sti 265 266 /* 267 * Do just enough of initialization to get to a working 268 * -ffreestanding environment and call tst_main(void). 269 */ 270 lea __preinit_array_start, %rdi 2711: 272 lea __preinit_array_end, %rsi 273 cmp %rdi, %rsi 274 je 2f 275 push %rdi 276 call *(%rdi) 277 pop %rdi 278 add $8, %rdi 279 jmp 1b 2802: 281 282 lea __init_array_start, %rdi 2831: 284 lea __init_array_end, %rsi 285 cmp %rdi, %rsi 286 je 2f 287 push %rdi 288 call *(%rdi) 289 pop %rdi 290 add $8, %rdi 291 jmp 1b 2922: 293 call main 294 jmp kvm_exit 295 296.global kvm_read_cregs 297kvm_read_cregs: 298 mov %cr0, %rax 299 mov %rax, (%rdi) 300 mov %cr2, %rax 301 mov %rax, 8(%rdi) 302 mov %cr3, %rax 303 mov %rax, 16(%rdi) 304 mov %cr4, %rax 305 mov %rax, 24(%rdi) 306 retq 307 308.global kvm_read_sregs 309kvm_read_sregs: 310 mov %cs, %ax 311 movw %ax, (%rdi) 312 mov %ds, %ax 313 movw %ax, 2(%rdi) 314 mov %es, %ax 315 movw %ax, 4(%rdi) 316 mov %fs, %ax 317 movw %ax, 6(%rdi) 318 mov %gs, %ax 319 movw %ax, 8(%rdi) 320 mov %ss, %ax 321 movw %ax, 10(%rdi) 322 retq 323 324handle_interrupt: 325 /* push CPU state */ 326 push %rbp 327 mov %rsp, %rbp 328 push %rax 329 push %rbx 330 push %rcx 331 push %rdx 332 push %rdi 333 push %rsi 334 push %r8 335 push %r9 336 push %r10 337 push %r11 338 339 /* load handler arguments from the stack and call handler */ 340 movq %rbp, %rdi 341 addq $24, %rdi 342 movq 8(%rbp), %rsi 343 movq 16(%rbp), %rdx 344 cld 345 call tst_handle_interrupt 346 347 /* restore CPU state and return */ 348 pop %r11 349 pop %r10 350 pop %r9 351 pop %r8 352 pop %rsi 353 pop %rdi 354 pop %rdx 355 pop %rcx 356 pop %rbx 357 pop %rax 358 pop %rbp 359 add $16, %rsp 360 iretq 361 362.macro create_intr_handler vector:req padargs=0 363.if \padargs 364 pushq $0 /* push dummy error code */ 365.endif 366 pushq $\vector 367 jmp handle_interrupt 368.endm 369 370.global kvm_handle_zerodiv 371kvm_handle_zerodiv: 372 create_intr_handler 0, padargs=1 373 374.global kvm_handle_debug 375kvm_handle_debug: 376 create_intr_handler 1, padargs=1 377 378.global kvm_handle_nmi 379kvm_handle_nmi: 380 create_intr_handler 2, padargs=1 381 382.global kvm_handle_breakpoint 383kvm_handle_breakpoint: 384 create_intr_handler 3, padargs=1 385 386.global kvm_handle_overflow 387kvm_handle_overflow: 388 create_intr_handler 4, padargs=1 389 390.global kvm_handle_bound_range_exc 391kvm_handle_bound_range_exc: 392 create_intr_handler 5, padargs=1 393 394.global kvm_handle_bad_opcode 395kvm_handle_bad_opcode: 396 create_intr_handler 6, padargs=1 397 398.global kvm_handle_device_error 399kvm_handle_device_error: 400 create_intr_handler 7, padargs=1 401 402.global kvm_handle_double_fault 403kvm_handle_double_fault: 404 create_intr_handler 8 405 406.global kvm_handle_invalid_tss 407kvm_handle_invalid_tss: 408 create_intr_handler 10 409 410.global kvm_handle_segfault 411kvm_handle_segfault: 412 create_intr_handler 11 413 414.global kvm_handle_stack_fault 415kvm_handle_stack_fault: 416 create_intr_handler 12 417 418.global kvm_handle_gpf 419kvm_handle_gpf: 420 create_intr_handler 13 421 422.global kvm_handle_page_fault 423kvm_handle_page_fault: 424 create_intr_handler 14 425 426.global kvm_handle_fpu_error 427kvm_handle_fpu_error: 428 create_intr_handler 16, padargs=1 429 430.global kvm_handle_alignment_error 431kvm_handle_alignment_error: 432 create_intr_handler 17 433 434.global kvm_handle_machine_check 435kvm_handle_machine_check: 436 create_intr_handler 18, padargs=1 437 438.global kvm_handle_simd_error 439kvm_handle_simd_error: 440 create_intr_handler 19, padargs=1 441 442.global kvm_handle_virt_error 443kvm_handle_virt_error: 444 create_intr_handler 20, padargs=1 445 446.global kvm_handle_cpe 447kvm_handle_cpe: 448 create_intr_handler 21 449 450.global kvm_handle_hv_injection 451kvm_handle_hv_injection: 452 create_intr_handler 28, padargs=1 453 454.global kvm_handle_vmm_comm 455kvm_handle_vmm_comm: 456 create_intr_handler 29 457 458.global kvm_handle_security_error 459kvm_handle_security_error: 460 create_intr_handler 30 461 462.global kvm_handle_bad_exception 463kvm_handle_bad_exception: 464 create_intr_handler -1, padargs=1 465 466 467.global kvm_exit 468kvm_exit: 469 movq $RESULT_ADDRESS, %rdi 470 movl $KVM_TEXIT, (%rdi) 471 hlt 472 jmp kvm_exit 473 474.global kvm_yield 475kvm_yield: 476 hlt 477 ret 478 479.global kvm_svm_guest_entry 480kvm_svm_guest_entry: 481 call *%rax 4821: hlt 483 jmp 1b 484 485.global kvm_svm_vmrun 486kvm_svm_vmrun: 487 pushq %rbx 488 pushq %rbp 489 pushq %r12 490 pushq %r13 491 pushq %r14 492 pushq %r15 493 494 clgi 495 496 /* Save full host state */ 497 movq $MSR_VM_HSAVE_PA, %rcx 498 rdmsr 499 shlq $32, %rdx 500 orq %rdx, %rax 501 vmsave 502 pushq %rax 503 504 /* Load guest registers */ 505 pushq %rdi 506 movq (%rdi), %rax 507 /* %rax is loaded by vmrun from VMCB */ 508 movq 0x10(%rdi), %rbx 509 movq 0x18(%rdi), %rcx 510 movq 0x20(%rdi), %rdx 511 movq 0x30(%rdi), %rsi 512 movq 0x38(%rdi), %rbp 513 /* %rsp is loaded by vmrun from VMCB */ 514 movq 0x48(%rdi), %r8 515 movq 0x50(%rdi), %r9 516 movq 0x58(%rdi), %r10 517 movq 0x60(%rdi), %r11 518 movq 0x68(%rdi), %r12 519 movq 0x70(%rdi), %r13 520 movq 0x78(%rdi), %r14 521 movq 0x80(%rdi), %r15 522 movq 0x28(%rdi), %rdi 523 524 vmload 525 vmrun 526 vmsave 527 528 /* Save guest registers */ 529 movq %rdi, %rax 530 popq %rdi 531 movq %rbx, 0x10(%rdi) 532 movq %rcx, 0x18(%rdi) 533 movq %rdx, 0x20(%rdi) 534 /* %rax contains guest %rdi */ 535 movq %rax, 0x28(%rdi) 536 movq %rsi, 0x30(%rdi) 537 movq %rbp, 0x38(%rdi) 538 movq %r8, 0x48(%rdi) 539 movq %r9, 0x50(%rdi) 540 movq %r10, 0x58(%rdi) 541 movq %r11, 0x60(%rdi) 542 movq %r12, 0x68(%rdi) 543 movq %r13, 0x70(%rdi) 544 movq %r14, 0x78(%rdi) 545 movq %r15, 0x80(%rdi) 546 /* copy guest %rax and %rsp from VMCB*/ 547 movq (%rdi), %rsi 548 movq 0x5f8(%rsi), %rax 549 movq %rax, 0x08(%rdi) 550 movq 0x5d8(%rsi), %rax 551 movq %rax, 0x40(%rdi) 552 553 /* Reload host state */ 554 popq %rax 555 vmload 556 557 stgi 558 559 popq %r15 560 popq %r14 561 popq %r13 562 popq %r12 563 popq %rbp 564 popq %rbx 565 retq 566 567.section .bss.pgtables, "aw", @nobits 568.global kvm_pagetable 569kvm_pagetable: 570 .skip 4096 571 572kvm_pgtable_l2: 573 .skip 4096 574 575kvm_pgtable_l3: 576 .skip 4 * 4096 577 578kvm_pgtable_l4: 579 .skip 4 * 512 * 4096 580 581.section .data 582.align 8 583.global kvm_gdt 584kvm_gdt: 585 .8byte 0 586 gdt32_entry type=0x1a l=1 limit=0 g=0 /* Code segment long mode */ 587 .skip (KVM_GDT_SIZE-2)*8 /* TSS and other segment descriptors */ 588 589.Lgdt_end: 590.global kvm_gdt_desc 591kvm_gdt_desc: 592 .2byte .Lgdt_end - kvm_gdt - 1 593 .8byte kvm_gdt 594 595 596.section .bss.stack, "aw", @nobits 597.global kvm_stack_bottom 598kvm_stack_bottom: 599 .skip 2 * 4096 600.global kvm_stack_top 601kvm_stack_top: 602 603.section .bss.tss 604.global kvm_tss 605kvm_tss: 606 .skip 0x6C 607.Ltss_end: 608 609.section .bss 610.align 8 611.global kvm_idt 612kvm_idt: 613 .skip 16 * 256 614.Lidt_end: 615 616.section .data 617.align 8 618.global kvm_idt_desc 619kvm_idt_desc: 620 .2byte .Lidt_end - kvm_idt - 1 621 .8byte kvm_idt 622