1 /*
2 * Copyright (c) 2022 Google Inc. All rights reserved
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining
5 * a copy of this software and associated documentation files
6 * (the "Software"), to deal in the Software without restriction,
7 * including without limitation the rights to use, copy, modify, merge,
8 * publish, distribute, sublicense, and/or sell copies of the Software,
9 * and to permit persons to whom the Software is furnished to do so,
10 * subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be
13 * included in all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
18 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
19 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
20 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
21 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24 #include <arch/ops.h>
25 #include <kernel/vm.h>
26
27 /*
28 * This storeTags function is borrowed from scudo, and can be deleted once we
29 * have scudo in the kernel
30 */
31 typedef uintptr_t uptr;
32 #define DCHECK_EQ(x, y)
storeTags(uptr Begin,uptr End)33 static uptr storeTags(uptr Begin, uptr End)
34 {
35 DCHECK_EQ(0, Begin % 16);
36 uptr LineSize, Next, Tmp;
37 __asm__ __volatile__(
38 ".arch_extension memtag\n"
39
40 "// Compute the cache line size in bytes (DCZID_EL0 stores it as the log2\n"
41 "// of the number of 4-byte words) and bail out to the slow path if DCZID_EL0\n"
42 "// indicates that the DC instructions are unavailable.\n"
43 "DCZID .req %[Tmp]\n"
44 "mrs DCZID, dczid_el0\n"
45 "tbnz DCZID, #4, 3f\n"
46 "and DCZID, DCZID, #15\n"
47 "mov %[LineSize], #4\n"
48 "lsl %[LineSize], %[LineSize], DCZID\n"
49 ".unreq DCZID\n"
50
51 "// Our main loop doesn't handle the case where we don't need to perform any\n"
52 "// DC GZVA operations. If the size of our tagged region is less than\n"
53 "// twice the cache line size, bail out to the slow path since it's not\n"
54 "// guaranteed that we'll be able to do a DC GZVA.\n"
55 "Size .req %[Tmp]\n"
56 "sub Size, %[End], %[Cur]\n"
57 "cmp Size, %[LineSize], lsl #1\n"
58 "b.lt 3f\n"
59 ".unreq Size\n"
60
61 "LineMask .req %[Tmp]\n"
62 "sub LineMask, %[LineSize], #1\n"
63
64 "// STZG until the start of the next cache line.\n"
65 "orr %[Next], %[Cur], LineMask\n"
66 "1:\n"
67 "stzg %[Cur], [%[Cur]], #16\n"
68 "cmp %[Cur], %[Next]\n"
69 "b.lt 1b\n"
70
71 "// DC GZVA cache lines until we have no more full cache lines.\n"
72 "bic %[Next], %[End], LineMask\n"
73 ".unreq LineMask\n"
74 "2:\n"
75 "dc gzva, %[Cur]\n"
76 "add %[Cur], %[Cur], %[LineSize]\n"
77 "cmp %[Cur], %[Next]\n"
78 "b.lt 2b\n"
79
80 "// STZG until the end of the tagged region. This loop is also used to handle\n"
81 "// slow path cases.\n"
82 "3:\n"
83 "cmp %[Cur], %[End]\n"
84 "b.ge 4f\n"
85 "stzg %[Cur], [%[Cur]], #16\n"
86 "b 3b\n"
87
88 "4:\n"
89 : [Cur] "+&r"(Begin), [LineSize] "=&r"(LineSize), [Next] "=&r"(Next),
90 [Tmp] "=&r"(Tmp)
91 : [End] "r"(End)
92 : "memory");
93 DCHECK_EQ(0, Begin % 16);
94 return Begin;
95 }
96
arch_clear_pages_and_tags(vaddr_t addr,size_t size)97 void arch_clear_pages_and_tags(vaddr_t addr, size_t size)
98 {
99 DEBUG_ASSERT(IS_PAGE_ALIGNED(addr));
100 DEBUG_ASSERT(IS_PAGE_ALIGNED(size));
101 addr &= 0x00ffffffffffffff;
102 storeTags(addr, addr + size);
103 }
104
105 bool arm64_mte_enabled;
106
arch_tagging_enabled(void)107 bool arch_tagging_enabled(void)
108 {
109 return arm64_mte_enabled;
110 }
111
arm64_tagging_supported(void)112 bool arm64_tagging_supported(void)
113 {
114 uint64_t v = ARM64_READ_SYSREG(id_aa64pfr1_el1);
115 return ((v & 0xf00) >= 0x200);
116 }
117