mmu.c (6703B)
1 #include <os9/os9.h> 2 3 #include <assert.h> 4 #include <errno.h> 5 #include <string.h> 6 7 #include <libk.h> 8 9 #include "sysreg.h" 10 #include "arch.h" 11 12 /* TCR_EL1 */ 13 #define NS 1ull 14 #define OS 2ull 15 #define IS 3ull 16 #define NC 0ull 17 #define WBA 1ull 18 #define WTA 2ull 19 #define WT 3ull 20 #define G16K 1ull 21 #define G4K 2ull 22 #define G64K 3ull 23 24 #define I4GB 0ull 25 #define I64GB 1ull 26 #define I1TB 2ull 27 #define I4TB 3ull 28 #define I16TB 4ull 29 #define I256TB 5ull 30 31 #define TBI1 (1ull << 38) 32 #define TBI0 (1ull << 37) 33 #define AS (1ull << 36) 34 #define IPS(x) ((x) << 32) 35 #define TG1(x) (((x)&3) << 30) 36 #define SH1(x) ((x) << 28) 37 #define ORGN1(x) ((x) << 26) 38 #define IRGN1(x) ((x) << 24) 39 #define EDP1 (1ull << 23) 40 #define A1 (1ull << 22) 41 #define T1SZ(x) (((x)&63) << 16) 42 #define TG0(x) (((x)&3) << 14) 43 #define SH0(x) ((x) << 12) 44 #define ORGN0(x) ((x) << 10) 45 #define IRGN0(x) ((x) << 8) 46 #define EDP0 (1ull << 7) 47 #define T0SZ(x) (((x)&63) << 0) 48 49 /* MMU */ 50 #define RDWR 0 51 #define EL0 1 52 #define RD 2 53 54 55 /* table attributes */ 56 #define APTable(x) ((pte_t) (x) << 62) 57 #define UXNTable(x) ((pte_t) (x) << 60) 58 #define PXNTable(x) ((pte_t) (x) << 59) 59 60 /* page high attributes */ 61 #define UXN ((pte_t) 1 << 54) 62 #define PXN ((pte_t) 1 << 53) 63 #define CONT ((pte_t) 1 << 52) 64 65 /* page low attributes */ 66 /* 67 * TODO: Define macros for OS(), because we are using 68 * the same macros than in OSx() but Non shareable has 69 * a different representation 70 */ 71 #define NG(x) ((pte_t) (x) << 11) 72 #define AF(x) ((pte_t) (x) << 10) 73 #define SH(x) ((pte_t) (x) << 8) 74 #define AP(x) ((pte_t) (x) << 6) 75 #define IDX(x) ((pte_t) (x) << 2) 76 77 #define VASIZE 40 78 #define MASK(pa) ((pa) & (-1ull>>(64-VASIZE) & ~0ull<<12)) 79 #define INVALID 0 80 #define BLOCK 1 81 #define TABLE 3 82 #define PAGE 3 83 84 #define TINDEX(lvl, va) ((va)>>(39-(lvl)*9) & 0x1FF) 85 86 #define DEV 7 87 #define MEM 0 88 89 /* ID_AA64MMFR0_EL1 */ 90 #define TGRAN4(x) ((x)>>28 & 0xf) 91 #define TGRAN64(x) ((x)>>24 & 0xf) 92 #define TGRAN16(x) ((x>>20 & 0xf) 93 #define ASIDBits(x) ((x)>>4 & 0xf) 94 #define PAR(x) ((x) & 0xf) 95 96 #define ASID8 0 97 #define ASID16 2 98 99 /* SPSR */ 100 #define NF(x) ((unsigned long) (x) << 31) 101 #define ZF(x) ((unsigned long) (x) << 30) 102 #define CF(x) ((unsigned long) (x) << 29) 103 #define VF(x) ((unsigned long) (x) << 28) 104 #define SS(x) ((unsigned long) (x) << 21) 105 #define IL(x) ((unsigned long) (x) << 20) 106 107 #define DAIF(x) ((unsigned long) (x) << 6) 108 #define MODE(x) ((unsigned long) (x) << 0) 109 110 #define A64 0 111 #define A32 1 112 113 #define AARCH(x) ((unsigned long) (x) << 4) 114 115 #define EL0t 0 116 #define EL1t 4 117 #define EL1h 5 118 119 /* MAIR */ 120 #define DEVICE(x) (x) 121 #define NORMAL(c, t, a) ((c) | (t) | (a)) 122 123 #define nGnRnE 0ull 124 #define nGnRE 2ull 125 #define nGRE 4ull 126 #define GRE 8ull 127 128 /* 129 * TODO: Improve these macros 130 */ 131 #define ONC 4ull 132 #define OWT 0ull 133 #define OWB 4ull 134 135 #define NTRANSIENT 8ull 136 #define RALLOC 2ull 137 #define WALLOC 1ull 138 139 #define TRANSIENT 0ull 140 #define NRALLOC 0ull 141 #define NWALLOC 0ull 142 143 144 int inlowmem = 1; 145 146 Ptable * 147 initptable(Task *tp) 148 { 149 struct ptable *pt, *ppt; 150 Task *parent; 151 152 parent = gettask(tp->ppid); 153 if (!parent) 154 panic("orphan task at creation"); 155 156 pt = &tp->ptable; 157 ppt = &parent->ptable; 158 159 pt->high = ppt->high; 160 pt->low = ppt->low; 161 pt->assid = tp->pid; 162 163 return pt; 164 } 165 166 static void * 167 newpage(void) 168 { 169 char *bp; 170 171 if ((bp = allocb()) == NULL) 172 return NULL; 173 return memset(bp, 0, PAGESIZE); 174 } 175 176 static pte_t 177 newtable(void) 178 { 179 char *bp; 180 181 if ((bp = newpage()) == NULL) 182 return INVALID; 183 184 if (!inlowmem) 185 bp = PMEM(bp); 186 187 return (pte_t) bp; 188 } 189 190 static pte_t * 191 walker(uintptr_t va) 192 { 193 pte_t *bp, e; 194 int lvl; 195 pte_t ttbr; 196 197 ttbr = (va & 1ull<<63) ? sysrd(TTBR1_EL1) : sysrd(TTBR0_EL1); 198 bp = (pte_t *) MASK(ttbr); 199 va = MASK(va); 200 201 for (lvl = 0; lvl < 3; lvl++) { 202 int n = TINDEX(lvl, va); 203 204 e = bp[n]; 205 if (e == INVALID) { 206 if ((e = newtable()) == INVALID) 207 return NULL; 208 bp[n] = e | TABLE; 209 barrier(DATA); 210 } 211 bp = (pte_t *) MASK(e); 212 if (!inlowmem) 213 bp = VMEM(bp); 214 } 215 216 return &bp[TINDEX(3, va)]; 217 } 218 219 int 220 vmap(phyaddr_t pa, uintptr_t va, int perm) 221 { 222 static mutex_t m; 223 pte_t *p, attr; 224 225 attr = AF(1) | SH(OS) | PAGE; 226 attr |= (perm & MW) ? AP(RDWR) : AP(RD); 227 attr |= (perm & MD) ? IDX(DEV) : IDX(MEM); 228 229 if ((perm & MX) == 0) 230 attr |= UXN|PXN; 231 232 if (va & 1ull<<63) 233 attr |= EL0; 234 235 /* 236 * TODO: It does not need locking, it needs a exclusive load 237 * locking is needed if we are in a system 238 * with SMMUv3 and page tables are shared. 239 */ 240 lock(&m); 241 if ((p = walker(va)) == NULL) { 242 unlock(&m); 243 return -1; 244 } 245 246 /* 247 * Here is needed to do a break-before-make as 248 * the architecture specifies to avoid a race condition 249 * in multi processor systems. 250 */ 251 *p = INVALID; 252 invtlb(va); 253 barrier(DATA); 254 barrier(CODE); 255 256 *p = MASK(pa) | attr; 257 barrier(DATA); 258 unlock(&m); 259 260 dbg("page %p: %llx\n", p, MASK(pa) | attr); 261 return 0; 262 } 263 264 265 int 266 vunmap(phyaddr_t pa, uintptr_t va) 267 { 268 /* TODO */ 269 return 0; 270 } 271 272 273 274 void 275 taskmap(void) 276 { 277 uint64_t tcr; 278 279 tcr = sysrd(TCR_EL1); 280 tcr &= ~T0SZ(-1); 281 tcr |= T0SZ(24); 282 syswr(TCR_EL1, tcr); 283 barrier(CODE); 284 285 /* 286 * TODO: setup SCTLR_EL1.WXN 287 * TODO: setup TTBR0_EL1 288 */ 289 } 290 291 void 292 kernelmap(void) 293 { 294 Map *mp; 295 uint64_t tcr; 296 297 tcr = sysrd(TCR_EL1); 298 tcr &= ~T1SZ(-1); 299 tcr |= T1SZ(24); 300 syswr(TCR_EL1, tcr); 301 barrier(CODE); 302 303 /* 304 * FIXME: We are using a full page when 305 * we only need 16 bytes with 16 byte alignment. 306 */ 307 syswr(TTBR1_EL1, newtable()); 308 309 dbg("TTBR1_EL1=%llx\n", sysrd(TTBR1_EL1)); 310 311 for (mp = mach.maps; mp->siz != 0; mp++) { 312 if (mapseg(mp) < 0) 313 panic("no kernel map"); 314 } 315 barrier(DATA); 316 barrier(CODE); 317 } 318 319 void 320 initmap(void) 321 { 322 pte_t *bp, tbl; 323 phyaddr_t pa; 324 uint64_t tcr; 325 pte_t attr = CONT | AF(1) | SH(OS) | AP(RDWR) | IDX(DEV) | BLOCK; 326 327 tcr = sysrd(TCR_EL1); 328 tcr &= ~T0SZ(-1); 329 tcr |= T0SZ(32); 330 syswr(TCR_EL1, tcr); 331 barrier(CODE); 332 333 tbl = newtable(); 334 bp = (pte_t *) tbl; 335 for (pa = 0; pa < 4*GiB; pa += 1*GiB) 336 *bp++ = pa | attr; 337 338 syswr(TTBR0_EL1, tbl); 339 barrier(DATA); 340 barrier(CODE); 341 } 342 343 void 344 mmuon(void) 345 { 346 uint64_t sctlr; 347 348 sctlr = sysrd(SCTLR_EL1); 349 sctlr |= M; 350 syswr(SCTLR_EL1, sctlr); 351 barrier(CODE); 352 } 353 354 void 355 immu(void) 356 { 357 uint64_t tcr, mair, id; 358 359 id = sysrd(ID_AA64MMFR0_EL1); 360 if (TGRAN4(id) != 0 || PAR(id) < I1TB || ASIDBits(id) == ASID8) 361 panic("unsupported mmu"); 362 363 /* FIXME: Add Inner properties */ 364 mair = 365 DEVICE(nGnRnE) << 56 | 366 DEVICE(nGnRE) << 48 | 367 DEVICE(nGRE) << 40 | 368 DEVICE(GRE) << 32 | 369 NORMAL(OWT, TRANSIENT, RALLOC|WALLOC) << 24 | 370 NORMAL(OWB, TRANSIENT, RALLOC|WALLOC) << 16 | 371 NORMAL(OWT, NTRANSIENT, RALLOC|WALLOC) << 8 | 372 NORMAL(OWB, NTRANSIENT, RALLOC|WALLOC) << 0; 373 374 tcr = 375 IPS(I1TB) | 376 AS | 377 TG1(G4K) | TG0(G4K) | 378 SH1(OS) | SH0(OS) | 379 ORGN1(WBA) | ORGN0(WBA) | 380 IRGN1(WBA) | IRGN0(WBA); 381 382 invalltlb(); 383 syswr(MAIR_EL1, mair); 384 syswr(TCR_EL1, tcr); 385 barrier(CODE); 386 }