1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * linux/arch/arm/boot/compressed/head.S 4 * 5 * Copyright (C) 1996-2002 Russell King 6 * Copyright (C) 2004 Hyok S. Choi (MPU support) 7 */ 8#include <linux/linkage.h> 9#include <asm/assembler.h> 10#include <asm/v7m.h> 11 12#include "efi-header.S" 13 14 AR_CLASS( .arch armv7-a ) 15 M_CLASS( .arch armv7-m ) 16 17/* 18 * Debugging stuff 19 * 20 * Note that these macros must not contain any code which is not 21 * 100% relocatable. Any attempt to do so will result in a crash. 22 * Please select one of the following when turning on debugging. 23 */ 24#ifdef DEBUG 25 26#if defined(CONFIG_DEBUG_ICEDCC) 27 28#if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_V6K) || defined(CONFIG_CPU_V7) 29 .macro loadsp, rb, tmp1, tmp2 30 .endm 31 .macro writeb, ch, rb, tmp 32 mcr p14, 0, \ch, c0, c5, 0 33 .endm 34#elif defined(CONFIG_CPU_XSCALE) 35 .macro loadsp, rb, tmp1, tmp2 36 .endm 37 .macro writeb, ch, rb, tmp 38 mcr p14, 0, \ch, c8, c0, 0 39 .endm 40#else 41 .macro loadsp, rb, tmp1, tmp2 42 .endm 43 .macro writeb, ch, rb, tmp 44 mcr p14, 0, \ch, c1, c0, 0 45 .endm 46#endif 47 48#else 49 50#include CONFIG_DEBUG_LL_INCLUDE 51 52 .macro writeb, ch, rb, tmp 53#ifdef CONFIG_DEBUG_UART_FLOW_CONTROL 54 waituartcts \tmp, \rb 55#endif 56 waituarttxrdy \tmp, \rb 57 senduart \ch, \rb 58 busyuart \tmp, \rb 59 .endm 60 61#if defined(CONFIG_ARCH_SA1100) 62 .macro loadsp, rb, tmp1, tmp2 63 mov \rb, #0x80000000 @ physical base address 64#ifdef CONFIG_DEBUG_LL_SER3 65 add \rb, \rb, #0x00050000 @ Ser3 66#else 67 add \rb, \rb, #0x00010000 @ Ser1 68#endif 69 .endm 70#else 71 .macro loadsp, rb, tmp1, tmp2 72 addruart \rb, \tmp1, \tmp2 73 .endm 74#endif 75#endif 76#endif 77 78 .macro kputc,val 79 mov r0, \val 80 bl putc 81 .endm 82 83 .macro kphex,val,len 84 mov r0, \val 85 mov r1, #\len 86 bl phex 87 .endm 88 89 /* 90 * Debug kernel copy by printing the memory addresses involved 91 */ 92 .macro dbgkc, begin, end, cbegin, cend 93#ifdef DEBUG 94 kputc #'C' 95 kputc #':' 96 kputc #'0' 97 kputc #'x' 98 kphex \begin, 8 /* Start of compressed kernel */ 99 kputc #'-' 100 kputc #'0' 101 kputc #'x' 102 kphex \end, 8 /* End of compressed kernel */ 103 kputc #'-' 104 kputc #'>' 105 kputc #'0' 106 kputc #'x' 107 kphex \cbegin, 8 /* Start of kernel copy */ 108 kputc #'-' 109 kputc #'0' 110 kputc #'x' 111 kphex \cend, 8 /* End of kernel copy */ 112 kputc #'\n' 113#endif 114 .endm 115 116 /* 117 * Debug print of the final appended DTB location 118 */ 119 .macro dbgadtb, begin, size 120#ifdef DEBUG 121 kputc #'D' 122 kputc #'T' 123 kputc #'B' 124 kputc #':' 125 kputc #'0' 126 kputc #'x' 127 kphex \begin, 8 /* Start of appended DTB */ 128 kputc #' ' 129 kputc #'(' 130 kputc #'0' 131 kputc #'x' 132 kphex \size, 8 /* Size of appended DTB */ 133 kputc #')' 134 kputc #'\n' 135#endif 136 .endm 137 138 .macro enable_cp15_barriers, reg 139 mrc p15, 0, \reg, c1, c0, 0 @ read SCTLR 140 tst \reg, #(1 << 5) @ CP15BEN bit set? 141 bne .L_\@ 142 orr \reg, \reg, #(1 << 5) @ CP15 barrier instructions 143 mcr p15, 0, \reg, c1, c0, 0 @ write SCTLR 144 ARM( .inst 0xf57ff06f @ v7+ isb ) 145 THUMB( isb ) 146.L_\@: 147 .endm 148 149 /* 150 * The kernel build system appends the size of the 151 * decompressed kernel at the end of the compressed data 152 * in little-endian form. 153 */ 154 .macro get_inflated_image_size, res:req, tmp1:req, tmp2:req 155 adr_l \res, .Linflated_image_size_offset 156 ldr \tmp1, [\res] 157 add \tmp1, \tmp1, \res @ address of inflated image size 158 159 ldrb \res, [\tmp1] @ get_unaligned_le32 160 ldrb \tmp2, [\tmp1, #1] 161 orr \res, \res, \tmp2, lsl #8 162 ldrb \tmp2, [\tmp1, #2] 163 ldrb \tmp1, [\tmp1, #3] 164 orr \res, \res, \tmp2, lsl #16 165 orr \res, \res, \tmp1, lsl #24 166 .endm 167 168 .macro record_seed 169#ifdef CONFIG_RANDOMIZE_BASE 170 sub ip, r1, ip, ror #1 @ poor man's kaslr seed, will 171 sub ip, r2, ip, ror #2 @ be superseded by kaslr-seed 172 sub ip, r3, ip, ror #3 @ from /chosen if present 173 sub ip, r4, ip, ror #5 174 sub ip, r5, ip, ror #8 175 sub ip, r6, ip, ror #13 176 sub ip, r7, ip, ror #21 177 sub ip, r8, ip, ror #3 178 sub ip, r9, ip, ror #24 179 sub ip, r10, ip, ror #27 180 sub ip, r11, ip, ror #19 181 sub ip, r13, ip, ror #14 182 sub ip, r14, ip, ror #2 183 str_l ip, __kaslr_seed, r9 184#endif 185 .endm 186 187 .section ".start", "ax" 188/* 189 * sort out different calling conventions 190 */ 191 .align 192 /* 193 * Always enter in ARM state for CPUs that support the ARM ISA. 194 * As of today (2014) that's exactly the members of the A and R 195 * classes. 196 */ 197 AR_CLASS( .arm ) 198start: 199 .type start,#function 200 /* 201 * These 7 nops along with the 1 nop immediately below for 202 * !THUMB2 form 8 nops that make the compressed kernel bootable 203 * on legacy ARM systems that were assuming the kernel in a.out 204 * binary format. The boot loaders on these systems would 205 * jump 32 bytes into the image to skip the a.out header. 206 * with these 8 nops filling exactly 32 bytes, things still 207 * work as expected on these legacy systems. Thumb2 mode keeps 208 * 7 of the nops as it turns out that some boot loaders 209 * were patching the initial instructions of the kernel, i.e 210 * had started to exploit this "patch area". 211 */ 212 __initial_nops 213 .rept 5 214 __nop 215 .endr 216#ifndef CONFIG_THUMB2_KERNEL 217 __nop 218#else 219 AR_CLASS( sub pc, pc, #3 ) @ A/R: switch to Thumb2 mode 220 M_CLASS( nop.w ) @ M: already in Thumb2 mode 221 .thumb 222#endif 223 W(b) 1f 224 225 .word _magic_sig @ Magic numbers to help the loader 226 .word _magic_start @ absolute load/run zImage address 227 .word _magic_end @ zImage end address 228 .word 0x04030201 @ endianness flag 229 .word 0x45454545 @ another magic number to indicate 230 .word _magic_table @ additional data table 231 232 __EFI_HEADER 2331: 234 ARM_BE8( setend be ) @ go BE8 if compiled for BE8 235 record_seed 236 AR_CLASS( mrs r9, cpsr ) 237#ifdef CONFIG_ARM_VIRT_EXT 238 bl __hyp_stub_install @ get into SVC mode, reversibly 239#endif 240 mov r7, r1 @ save architecture ID 241 mov r8, r2 @ save atags pointer 242 243#ifndef CONFIG_CPU_V7M 244 /* 245 * Booting from Angel - need to enter SVC mode and disable 246 * FIQs/IRQs (numeric definitions from angel arm.h source). 247 * We only do this if we were in user mode on entry. 248 */ 249 mrs r2, cpsr @ get current mode 250 tst r2, #3 @ not user? 251 bne not_angel 252 mov r0, #0x17 @ angel_SWIreason_EnterSVC 253 ARM( swi 0x123456 ) @ angel_SWI_ARM 254 THUMB( svc 0xab ) @ angel_SWI_THUMB 255not_angel: 256 safe_svcmode_maskall r0 257 msr spsr_cxsf, r9 @ Save the CPU boot mode in 258 @ SPSR 259#endif 260 /* 261 * Note that some cache flushing and other stuff may 262 * be needed here - is there an Angel SWI call for this? 263 */ 264 265 /* 266 * some architecture specific code can be inserted 267 * by the linker here, but it should preserve r7, r8, and r9. 268 */ 269 270 .text 271 272#ifdef CONFIG_AUTO_ZRELADDR 273 /* 274 * Find the start of physical memory. As we are executing 275 * without the MMU on, we are in the physical address space. 276 * We just need to get rid of any offset by aligning the 277 * address. 278 * 279 * This alignment is a balance between the requirements of 280 * different platforms - we have chosen 128MB to allow 281 * platforms which align the start of their physical memory 282 * to 128MB to use this feature, while allowing the zImage 283 * to be placed within the first 128MB of memory on other 284 * platforms. Increasing the alignment means we place 285 * stricter alignment requirements on the start of physical 286 * memory, but relaxing it means that we break people who 287 * are already placing their zImage in (eg) the top 64MB 288 * of this range. 289 */ 290 mov r4, pc 291 and r4, r4, #0xf8000000 292 /* Determine final kernel image address. */ 293 add r4, r4, #TEXT_OFFSET 294#else 295 ldr r4, =zreladdr 296#endif 297 298 /* 299 * Set up a page table only if it won't overwrite ourself. 300 * That means r4 < pc || r4 - 16k page directory > &_end. 301 * Given that r4 > &_end is most unfrequent, we add a rough 302 * additional 1MB of room for a possible appended DTB. 303 */ 304 mov r0, pc 305 cmp r0, r4 306 ldrcc r0, .Lheadroom 307 addcc r0, r0, pc 308 cmpcc r4, r0 309 orrcc r4, r4, #1 @ remember we skipped cache_on 310 blcs cache_on 311 312restart: adr_l r0, LC1 313 ldr sp, [r0] 314 ldr r6, [r0, #4] 315 add sp, sp, r0 316 add r6, r6, r0 317 318 get_inflated_image_size r9, r10, lr 319 320#ifndef CONFIG_ZBOOT_ROM 321 /* malloc space is above the relocated stack (64k max) */ 322 add r10, sp, #MALLOC_SIZE 323#else 324 /* 325 * With ZBOOT_ROM the bss/stack is non relocatable, 326 * but someone could still run this code from RAM, 327 * in which case our reference is _edata. 328 */ 329 mov r10, r6 330#endif 331 332 mov r5, #0 @ init dtb size to 0 333#ifdef CONFIG_ARM_APPENDED_DTB 334/* 335 * r4 = final kernel address (possibly with LSB set) 336 * r5 = appended dtb size (still unknown) 337 * r6 = _edata 338 * r7 = architecture ID 339 * r8 = atags/device tree pointer 340 * r9 = size of decompressed image 341 * r10 = end of this image, including bss/stack/malloc space if non XIP 342 * sp = stack pointer 343 * 344 * if there are device trees (dtb) appended to zImage, advance r10 so that the 345 * dtb data will get relocated along with the kernel if necessary. 346 */ 347 348 ldr lr, [r6, #0] 349#ifndef __ARMEB__ 350 ldr r1, =0xedfe0dd0 @ sig is 0xd00dfeed big endian 351#else 352 ldr r1, =0xd00dfeed 353#endif 354 cmp lr, r1 355 bne dtb_check_done @ not found 356 357#ifdef CONFIG_ARM_ATAG_DTB_COMPAT 358 /* 359 * OK... Let's do some funky business here. 360 * If we do have a DTB appended to zImage, and we do have 361 * an ATAG list around, we want the later to be translated 362 * and folded into the former here. No GOT fixup has occurred 363 * yet, but none of the code we're about to call uses any 364 * global variable. 365 */ 366 367 /* Get the initial DTB size */ 368 ldr r5, [r6, #4] 369#ifndef __ARMEB__ 370 /* convert to little endian */ 371 eor r1, r5, r5, ror #16 372 bic r1, r1, #0x00ff0000 373 mov r5, r5, ror #8 374 eor r5, r5, r1, lsr #8 375#endif 376 dbgadtb r6, r5 377 /* 50% DTB growth should be good enough */ 378 add r5, r5, r5, lsr #1 379 /* preserve 64-bit alignment */ 380 add r5, r5, #7 381 bic r5, r5, #7 382 /* clamp to 32KB min and 1MB max */ 383 cmp r5, #(1 << 15) 384 movlo r5, #(1 << 15) 385 cmp r5, #(1 << 20) 386 movhi r5, #(1 << 20) 387 /* temporarily relocate the stack past the DTB work space */ 388 add sp, sp, r5 389 390 mov r0, r8 391 mov r1, r6 392 mov r2, r5 393 bl atags_to_fdt 394 395 /* 396 * If returned value is 1, there is no ATAG at the location 397 * pointed by r8. Try the typical 0x100 offset from start 398 * of RAM and hope for the best. 399 */ 400 cmp r0, #1 401 sub r0, r4, #TEXT_OFFSET 402 bic r0, r0, #1 403 add r0, r0, #0x100 404 mov r1, r6 405 mov r2, r5 406 bleq atags_to_fdt 407 408 sub sp, sp, r5 409#endif 410 411 mov r8, r6 @ use the appended device tree 412 413 /* 414 * Make sure that the DTB doesn't end up in the final 415 * kernel's .bss area. To do so, we adjust the decompressed 416 * kernel size to compensate if that .bss size is larger 417 * than the relocated code. 418 */ 419 ldr r5, =_kernel_bss_size 420 adr r1, wont_overwrite 421 sub r1, r6, r1 422 subs r1, r5, r1 423 addhi r9, r9, r1 424 425 /* Get the current DTB size */ 426 ldr r5, [r6, #4] 427#ifndef __ARMEB__ 428 /* convert r5 (dtb size) to little endian */ 429 eor r1, r5, r5, ror #16 430 bic r1, r1, #0x00ff0000 431 mov r5, r5, ror #8 432 eor r5, r5, r1, lsr #8 433#endif 434 435 /* preserve 64-bit alignment */ 436 add r5, r5, #7 437 bic r5, r5, #7 438 439 /* relocate some pointers past the appended dtb */ 440 add r6, r6, r5 441 add r10, r10, r5 442 add sp, sp, r5 443dtb_check_done: 444#endif 445 446#ifdef CONFIG_RANDOMIZE_BASE 447 ldr r1, __kaslr_offset @ check if the kaslr_offset is 448 cmp r1, #0 @ already set 449 bne 1f 450 451 stmfd sp!, {r0-r3, ip, lr} 452#ifdef CONFIG_ARCH_HISI 453#ifdef CONFIG_ARM_APPENDED_DTB 454#ifdef CONFIG_START_MEM_2M_ALIGN 455 mov r0, r4 456#ifdef CONFIG_CORTEX_A9 457 lsr r0, r0, #20 458 lsl r0, r0, #20 459#else 460 lsr r0, r0, #21 461 lsl r0, r0, #21 462#endif 463 add r0, r0, #0x1000 464 ldr r1, [r0] 465#ifndef __ARMEB__ 466 ldr r2, =0xedfe0dd0 @ sig is 0xd00dfeed big endian 467#else 468 ldr r2, =0xd00dfeed 469#endif 470 cmp r1, r2 471 moveq r8, r0 472#endif 473#endif 474#endif 475 adr_l r2, _text @ start of zImage 476 stmfd sp!, {r2, r8, r10} @ pass stack arguments 477 478 ldr_l r3, __kaslr_seed 479#if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_V6K) || defined(CONFIG_CPU_V7) 480 /* 481 * Get some pseudo-entropy from the low bits of the generic 482 * timer if it is implemented. 483 */ 484 mrc p15, 0, r1, c0, c1, 1 @ read ID_PFR1 register 485 tst r1, #0x10000 @ have generic timer? 486 mrrcne p15, 1, r3, r1, c14 @ read CNTVCT 487#endif 488 adr_l r0, __kaslr_offset @ pass &__kaslr_offset in r0 489 mov r1, r4 @ pass base address 490 mov r2, r9 @ pass decompressed image size 491 eor r3, r3, r3, ror #16 @ pass pseudorandom seed 492 bl kaslr_early_init 493 add sp, sp, #12 494 cmp r0, #0 495 addne r4, r4, r0 @ add offset to base address 496#ifdef CONFIG_VXBOOT 497#ifdef CONFIG_START_MEM_2M_ALIGN 498#ifdef CONFIG_CORTEX_A9 499 adr r1, vx_edata 500 strne r6, [r1] 501#endif 502#endif 503#endif 504 ldmfd sp!, {r0-r3, ip, lr} 505 bne restart 5061: 507#endif 508 509/* 510 * Check to see if we will overwrite ourselves. 511 * r4 = final kernel address (possibly with LSB set) 512 * r9 = size of decompressed image 513 * r10 = end of this image, including bss/stack/malloc space if non XIP 514 * We basically want: 515 * r4 - 16k page directory >= r10 -> OK 516 * r4 + image length <= address of wont_overwrite -> OK 517 * Note: the possible LSB in r4 is harmless here. 518 */ 519 add r10, r10, #16384 520 cmp r4, r10 521 bhs wont_overwrite 522 add r10, r4, r9 523 adr r9, wont_overwrite 524 cmp r10, r9 525 bls wont_overwrite 526 527/* 528 * Relocate ourselves past the end of the decompressed kernel. 529 * r6 = _edata 530 * r10 = end of the decompressed kernel 531 * Because we always copy ahead, we need to do it from the end and go 532 * backward in case the source and destination overlap. 533 */ 534 /* 535 * Bump to the next 256-byte boundary with the size of 536 * the relocation code added. This avoids overwriting 537 * ourself when the offset is small. 538 */ 539 add r10, r10, #((reloc_code_end - restart + 256) & ~255) 540 bic r10, r10, #255 541 542 /* Get start of code we want to copy and align it down. */ 543 adr r5, restart 544 bic r5, r5, #31 545 546/* Relocate the hyp vector base if necessary */ 547#ifdef CONFIG_ARM_VIRT_EXT 548 mrs r0, spsr 549 and r0, r0, #MODE_MASK 550 cmp r0, #HYP_MODE 551 bne 1f 552 553 /* 554 * Compute the address of the hyp vectors after relocation. 555 * Call __hyp_set_vectors with the new address so that we 556 * can HVC again after the copy. 557 */ 558 adr_l r0, __hyp_stub_vectors 559 sub r0, r0, r5 560 add r0, r0, r10 561 bl __hyp_set_vectors 5621: 563#endif 564 565 sub r9, r6, r5 @ size to copy 566 add r9, r9, #31 @ rounded up to a multiple 567 bic r9, r9, #31 @ ... of 32 bytes 568 add r6, r9, r5 569 add r9, r9, r10 570 571#ifdef DEBUG 572 sub r10, r6, r5 573 sub r10, r9, r10 574 /* 575 * We are about to copy the kernel to a new memory area. 576 * The boundaries of the new memory area can be found in 577 * r10 and r9, whilst r5 and r6 contain the boundaries 578 * of the memory we are going to copy. 579 * Calling dbgkc will help with the printing of this 580 * information. 581 */ 582 dbgkc r5, r6, r10, r9 583#endif 584 5851: ldmdb r6!, {r0 - r3, r10 - r12, lr} 586 cmp r6, r5 587 stmdb r9!, {r0 - r3, r10 - r12, lr} 588 bhi 1b 589 590 /* Preserve offset to relocated code. */ 591 sub r6, r9, r6 592 593 mov r0, r9 @ start of relocated zImage 594 add r1, sp, r6 @ end of relocated zImage 595 bl cache_clean_flush 596 597 badr r0, restart 598 add r0, r0, r6 599 mov pc, r0 600 601wont_overwrite: 602 adr r0, LC0 603 ldmia r0, {r1, r2, r3, r11, r12} 604 sub r0, r0, r1 @ calculate the delta offset 605 606/* 607 * If delta is zero, we are running at the address we were linked at. 608 * r0 = delta 609 * r2 = BSS start 610 * r3 = BSS end 611 * r4 = kernel execution address (possibly with LSB set) 612 * r5 = appended dtb size (0 if not present) 613 * r7 = architecture ID 614 * r8 = atags pointer 615 * r11 = GOT start 616 * r12 = GOT end 617 * sp = stack pointer 618 */ 619 orrs r1, r0, r5 620 beq not_relocated 621 622 add r11, r11, r0 623 add r12, r12, r0 624 625#ifndef CONFIG_ZBOOT_ROM 626 /* 627 * If we're running fully PIC === CONFIG_ZBOOT_ROM = n, 628 * we need to fix up pointers into the BSS region. 629 * Note that the stack pointer has already been fixed up. 630 */ 631 add r2, r2, r0 632 add r3, r3, r0 633 634 /* 635 * Relocate all entries in the GOT table. 636 * Bump bss entries to _edata + dtb size 637 */ 6381: ldr r1, [r11, #0] @ relocate entries in the GOT 639 add r1, r1, r0 @ This fixes up C references 640 cmp r1, r2 @ if entry >= bss_start && 641 cmphs r3, r1 @ bss_end > entry 642 addhi r1, r1, r5 @ entry += dtb size 643 str r1, [r11], #4 @ next entry 644 cmp r11, r12 645 blo 1b 646 647 /* bump our bss pointers too */ 648 add r2, r2, r5 649 add r3, r3, r5 650 651#else 652 653 /* 654 * Relocate entries in the GOT table. We only relocate 655 * the entries that are outside the (relocated) BSS region. 656 */ 6571: ldr r1, [r11, #0] @ relocate entries in the GOT 658 cmp r1, r2 @ entry < bss_start || 659 cmphs r3, r1 @ _end < entry 660 addlo r1, r1, r0 @ table. This fixes up the 661 str r1, [r11], #4 @ C references. 662 cmp r11, r12 663 blo 1b 664#endif 665 666not_relocated: mov r0, #0 6671: str r0, [r2], #4 @ clear bss 668 str r0, [r2], #4 669 str r0, [r2], #4 670 str r0, [r2], #4 671 cmp r2, r3 672 blo 1b 673 674 /* 675 * Did we skip the cache setup earlier? 676 * That is indicated by the LSB in r4. 677 * Do it now if so. 678 */ 679 tst r4, #1 680 bic r4, r4, #1 681 blne cache_on 682 683/* 684 * The C runtime environment should now be setup sufficiently. 685 * Set up some pointers, and start decompressing. 686 * r4 = kernel execution address 687 * r7 = architecture ID 688 * r8 = atags pointer 689 */ 690 mov r0, r4 691 mov r1, sp @ malloc space above stack 692 add r2, sp, #MALLOC_SIZE @ 64k max 693 mov r3, r7 694 bl decompress_kernel 695 696 get_inflated_image_size r1, r2, r3 697 698 mov r0, r4 @ start of inflated image 699 add r1, r1, r0 @ end of inflated image 700 bl cache_clean_flush 701 bl cache_off 702 703#ifdef CONFIG_ARM_VIRT_EXT 704 mrs r0, spsr @ Get saved CPU boot mode 705 and r0, r0, #MODE_MASK 706 cmp r0, #HYP_MODE @ if not booted in HYP mode... 707 bne __enter_kernel @ boot kernel directly 708 709 adr_l r0, __hyp_reentry_vectors 710 bl __hyp_set_vectors 711 __HVC(0) @ otherwise bounce to hyp mode 712 713 b . @ should never be reached 714#else 715 b __enter_kernel 716#endif 717 718 .align 2 719 .type LC0, #object 720LC0: .word LC0 @ r1 721 .word __bss_start @ r2 722 .word _end @ r3 723 .word _got_start @ r11 724 .word _got_end @ ip 725 .size LC0, . - LC0 726 727 .type LC1, #object 728LC1: .word .L_user_stack_end - LC1 @ sp 729 .word _edata - LC1 @ r6 730 .size LC1, . - LC1 731 732.Lheadroom: 733 .word _end - restart + 16384 + 1024*1024 734 735.Linflated_image_size_offset: 736 .long (input_data_end - 4) - . 737 738#ifdef CONFIG_ARCH_RPC 739 .globl params 740params: ldr r0, =0x10000100 @ params_phys for RPC 741 mov pc, lr 742 .ltorg 743 .align 744#endif 745 746/* 747 * dcache_line_size - get the minimum D-cache line size from the CTR register 748 * on ARMv7. 749 */ 750 .macro dcache_line_size, reg, tmp 751#ifdef CONFIG_CPU_V7M 752 movw \tmp, #:lower16:BASEADDR_V7M_SCB + V7M_SCB_CTR 753 movt \tmp, #:upper16:BASEADDR_V7M_SCB + V7M_SCB_CTR 754 ldr \tmp, [\tmp] 755#else 756 mrc p15, 0, \tmp, c0, c0, 1 @ read ctr 757#endif 758 lsr \tmp, \tmp, #16 759 and \tmp, \tmp, #0xf @ cache line size encoding 760 mov \reg, #4 @ bytes per word 761 mov \reg, \reg, lsl \tmp @ actual cache line size 762 .endm 763 764/* 765 * Turn on the cache. We need to setup some page tables so that we 766 * can have both the I and D caches on. 767 * 768 * We place the page tables 16k down from the kernel execution address, 769 * and we hope that nothing else is using it. If we're using it, we 770 * will go pop! 771 * 772 * On entry, 773 * r4 = kernel execution address 774 * r7 = architecture number 775 * r8 = atags pointer 776 * On exit, 777 * r0, r1, r2, r3, r9, r10, r12 corrupted 778 * This routine must preserve: 779 * r4, r7, r8 780 */ 781 .align 5 782cache_on: mov r3, #8 @ cache_on function 783 b call_cache_fn 784 785/* 786 * Initialize the highest priority protection region, PR7 787 * to cover all 32bit address and cacheable and bufferable. 788 */ 789__armv4_mpu_cache_on: 790 mov r0, #0x3f @ 4G, the whole 791 mcr p15, 0, r0, c6, c7, 0 @ PR7 Area Setting 792 mcr p15, 0, r0, c6, c7, 1 793 794 mov r0, #0x80 @ PR7 795 mcr p15, 0, r0, c2, c0, 0 @ D-cache on 796 mcr p15, 0, r0, c2, c0, 1 @ I-cache on 797 mcr p15, 0, r0, c3, c0, 0 @ write-buffer on 798 799 mov r0, #0xc000 800 mcr p15, 0, r0, c5, c0, 1 @ I-access permission 801 mcr p15, 0, r0, c5, c0, 0 @ D-access permission 802 803 mov r0, #0 804 mcr p15, 0, r0, c7, c10, 4 @ drain write buffer 805 mcr p15, 0, r0, c7, c5, 0 @ flush(inval) I-Cache 806 mcr p15, 0, r0, c7, c6, 0 @ flush(inval) D-Cache 807 mrc p15, 0, r0, c1, c0, 0 @ read control reg 808 @ ...I .... ..D. WC.M 809 orr r0, r0, #0x002d @ .... .... ..1. 11.1 810 orr r0, r0, #0x1000 @ ...1 .... .... .... 811 812 mcr p15, 0, r0, c1, c0, 0 @ write control reg 813 814 mov r0, #0 815 mcr p15, 0, r0, c7, c5, 0 @ flush(inval) I-Cache 816 mcr p15, 0, r0, c7, c6, 0 @ flush(inval) D-Cache 817 mov pc, lr 818 819__armv3_mpu_cache_on: 820 mov r0, #0x3f @ 4G, the whole 821 mcr p15, 0, r0, c6, c7, 0 @ PR7 Area Setting 822 823 mov r0, #0x80 @ PR7 824 mcr p15, 0, r0, c2, c0, 0 @ cache on 825 mcr p15, 0, r0, c3, c0, 0 @ write-buffer on 826 827 mov r0, #0xc000 828 mcr p15, 0, r0, c5, c0, 0 @ access permission 829 830 mov r0, #0 831 mcr p15, 0, r0, c7, c0, 0 @ invalidate whole cache v3 832 /* 833 * ?? ARMv3 MMU does not allow reading the control register, 834 * does this really work on ARMv3 MPU? 835 */ 836 mrc p15, 0, r0, c1, c0, 0 @ read control reg 837 @ .... .... .... WC.M 838 orr r0, r0, #0x000d @ .... .... .... 11.1 839 /* ?? this overwrites the value constructed above? */ 840 mov r0, #0 841 mcr p15, 0, r0, c1, c0, 0 @ write control reg 842 843 /* ?? invalidate for the second time? */ 844 mcr p15, 0, r0, c7, c0, 0 @ invalidate whole cache v3 845 mov pc, lr 846 847#ifdef CONFIG_CPU_DCACHE_WRITETHROUGH 848#define CB_BITS 0x08 849#else 850#define CB_BITS 0x0c 851#endif 852 853__setup_mmu: sub r3, r4, #16384 @ Page directory size 854 bic r3, r3, #0xff @ Align the pointer 855 bic r3, r3, #0x3f00 856/* 857 * Initialise the page tables, turning on the cacheable and bufferable 858 * bits for the RAM area only. 859 */ 860 mov r0, r3 861 mov r9, r0, lsr #18 862 mov r9, r9, lsl #18 @ start of RAM 863 add r10, r9, #0x10000000 @ a reasonable RAM size 864 mov r1, #0x12 @ XN|U + section mapping 865 orr r1, r1, #3 << 10 @ AP=11 866 add r2, r3, #16384 8671: cmp r1, r9 @ if virt > start of RAM 868 cmphs r10, r1 @ && end of RAM > virt 869 bic r1, r1, #0x1c @ clear XN|U + C + B 870 orrlo r1, r1, #0x10 @ Set XN|U for non-RAM 871 orrhs r1, r1, r6 @ set RAM section settings 872 str r1, [r0], #4 @ 1:1 mapping 873 add r1, r1, #1048576 874 teq r0, r2 875 bne 1b 876/* 877 * Make sure our entire executable image (including payload) is mapped 878 * cacheable, in case it is located outside the region we covered above. 879 * (This may be the case if running from flash or with randomization enabled) 880 * If the regions happen to overlap, we just duplicate some of the above. 881 */ 882 orr r1, r6, #0x04 @ ensure B is set for this 883 orr r1, r1, #3 << 10 884 mov r2, pc 885 adr_l r9, _end 886 mov r2, r2, lsr #20 887 mov r9, r9, lsr #20 888 orr r1, r1, r2, lsl #20 889 add r0, r3, r2, lsl #2 890 add r9, r3, r9, lsl #2 8910: str r1, [r0], #4 892 add r1, r1, #1048576 893 cmp r0, r9 894 bls 0b 895 mov pc, lr 896ENDPROC(__setup_mmu) 897 898@ Enable unaligned access on v6, to allow better code generation 899@ for the decompressor C code: 900__armv6_mmu_cache_on: 901 mrc p15, 0, r0, c1, c0, 0 @ read SCTLR 902 bic r0, r0, #2 @ A (no unaligned access fault) 903 orr r0, r0, #1 << 22 @ U (v6 unaligned access model) 904 mcr p15, 0, r0, c1, c0, 0 @ write SCTLR 905 b __armv4_mmu_cache_on 906 907__arm926ejs_mmu_cache_on: 908#ifdef CONFIG_CPU_DCACHE_WRITETHROUGH 909 mov r0, #4 @ put dcache in WT mode 910 mcr p15, 7, r0, c15, c0, 0 911#endif 912 913__armv4_mmu_cache_on: 914 mov r12, lr 915#ifdef CONFIG_MMU 916 mov r6, #CB_BITS | 0x12 @ U 917 bl __setup_mmu 918 mov r0, #0 919 mcr p15, 0, r0, c7, c10, 4 @ drain write buffer 920 mcr p15, 0, r0, c8, c7, 0 @ flush I,D TLBs 921 mrc p15, 0, r0, c1, c0, 0 @ read control reg 922 orr r0, r0, #0x5000 @ I-cache enable, RR cache replacement 923 orr r0, r0, #0x0030 924 ARM_BE8( orr r0, r0, #1 << 25 ) @ big-endian page tables 925 bl __common_mmu_cache_on 926 mov r0, #0 927 mcr p15, 0, r0, c8, c7, 0 @ flush I,D TLBs 928#endif 929 mov pc, r12 930 931__armv7_mmu_cache_on: 932 enable_cp15_barriers r11 933 mov r12, lr 934#ifdef CONFIG_MMU 935 mrc p15, 0, r11, c0, c1, 4 @ read ID_MMFR0 936 tst r11, #0xf @ VMSA 937 movne r6, #CB_BITS | 0x02 @ !XN 938 blne __setup_mmu 939 mov r0, #0 940 mcr p15, 0, r0, c7, c10, 4 @ drain write buffer 941 tst r11, #0xf @ VMSA 942 mcrne p15, 0, r0, c8, c7, 0 @ flush I,D TLBs 943#endif 944 mrc p15, 0, r0, c1, c0, 0 @ read control reg 945 bic r0, r0, #1 << 28 @ clear SCTLR.TRE 946 orr r0, r0, #0x5000 @ I-cache enable, RR cache replacement 947 orr r0, r0, #0x003c @ write buffer 948 bic r0, r0, #2 @ A (no unaligned access fault) 949 orr r0, r0, #1 << 22 @ U (v6 unaligned access model) 950 @ (needed for ARM1176) 951#ifdef CONFIG_MMU 952 ARM_BE8( orr r0, r0, #1 << 25 ) @ big-endian page tables 953 mrcne p15, 0, r6, c2, c0, 2 @ read ttb control reg 954 orrne r0, r0, #1 @ MMU enabled 955 movne r1, #0xfffffffd @ domain 0 = client 956 bic r6, r6, #1 << 31 @ 32-bit translation system 957 bic r6, r6, #(7 << 0) | (1 << 4) @ use only ttbr0 958 mcrne p15, 0, r3, c2, c0, 0 @ load page table pointer 959 mcrne p15, 0, r1, c3, c0, 0 @ load domain access control 960 mcrne p15, 0, r6, c2, c0, 2 @ load ttb control 961#endif 962 mcr p15, 0, r0, c7, c5, 4 @ ISB 963 mcr p15, 0, r0, c1, c0, 0 @ load control register 964 mrc p15, 0, r0, c1, c0, 0 @ and read it back 965 mov r0, #0 966 mcr p15, 0, r0, c7, c5, 4 @ ISB 967 mov pc, r12 968 969__fa526_cache_on: 970 mov r12, lr 971 mov r6, #CB_BITS | 0x12 @ U 972 bl __setup_mmu 973 mov r0, #0 974 mcr p15, 0, r0, c7, c7, 0 @ Invalidate whole cache 975 mcr p15, 0, r0, c7, c10, 4 @ drain write buffer 976 mcr p15, 0, r0, c8, c7, 0 @ flush UTLB 977 mrc p15, 0, r0, c1, c0, 0 @ read control reg 978 orr r0, r0, #0x1000 @ I-cache enable 979 bl __common_mmu_cache_on 980 mov r0, #0 981 mcr p15, 0, r0, c8, c7, 0 @ flush UTLB 982 mov pc, r12 983 984__common_mmu_cache_on: 985#ifndef CONFIG_THUMB2_KERNEL 986#ifndef DEBUG 987 orr r0, r0, #0x000d @ Write buffer, mmu 988#endif 989 mov r1, #-1 990 mcr p15, 0, r3, c2, c0, 0 @ load page table pointer 991 mcr p15, 0, r1, c3, c0, 0 @ load domain access control 992 b 1f 993 .align 5 @ cache line aligned 9941: mcr p15, 0, r0, c1, c0, 0 @ load control register 995 mrc p15, 0, r0, c1, c0, 0 @ and read it back to 996 sub pc, lr, r0, lsr #32 @ properly flush pipeline 997#endif 998 999#define PROC_ENTRY_SIZE (4*5) 1000 1001/* 1002 * Here follow the relocatable cache support functions for the 1003 * various processors. This is a generic hook for locating an 1004 * entry and jumping to an instruction at the specified offset 1005 * from the start of the block. Please note this is all position 1006 * independent code. 1007 * 1008 * r1 = corrupted 1009 * r2 = corrupted 1010 * r3 = block offset 1011 * r9 = corrupted 1012 * r12 = corrupted 1013 */ 1014 1015call_cache_fn: adr r12, proc_types 1016#ifdef CONFIG_CPU_CP15 1017 mrc p15, 0, r9, c0, c0 @ get processor ID 1018#elif defined(CONFIG_CPU_V7M) 1019 /* 1020 * On v7-M the processor id is located in the V7M_SCB_CPUID 1021 * register, but as cache handling is IMPLEMENTATION DEFINED on 1022 * v7-M (if existant at all) we just return early here. 1023 * If V7M_SCB_CPUID were used the cpu ID functions (i.e. 1024 * __armv7_mmu_cache_{on,off,flush}) would be selected which 1025 * use cp15 registers that are not implemented on v7-M. 1026 */ 1027 bx lr 1028#else 1029 ldr r9, =CONFIG_PROCESSOR_ID 1030#endif 10311: ldr r1, [r12, #0] @ get value 1032 ldr r2, [r12, #4] @ get mask 1033 eor r1, r1, r9 @ (real ^ match) 1034 tst r1, r2 @ & mask 1035 ARM( addeq pc, r12, r3 ) @ call cache function 1036 THUMB( addeq r12, r3 ) 1037 THUMB( moveq pc, r12 ) @ call cache function 1038 add r12, r12, #PROC_ENTRY_SIZE 1039 b 1b 1040 1041/* 1042 * Table for cache operations. This is basically: 1043 * - CPU ID match 1044 * - CPU ID mask 1045 * - 'cache on' method instruction 1046 * - 'cache off' method instruction 1047 * - 'cache flush' method instruction 1048 * 1049 * We match an entry using: ((real_id ^ match) & mask) == 0 1050 * 1051 * Writethrough caches generally only need 'on' and 'off' 1052 * methods. Writeback caches _must_ have the flush method 1053 * defined. 1054 */ 1055 .align 2 1056 .type proc_types,#object 1057proc_types: 1058 .word 0x41000000 @ old ARM ID 1059 .word 0xff00f000 1060 mov pc, lr 1061 THUMB( nop ) 1062 mov pc, lr 1063 THUMB( nop ) 1064 mov pc, lr 1065 THUMB( nop ) 1066 1067 .word 0x41007000 @ ARM7/710 1068 .word 0xfff8fe00 1069 mov pc, lr 1070 THUMB( nop ) 1071 mov pc, lr 1072 THUMB( nop ) 1073 mov pc, lr 1074 THUMB( nop ) 1075 1076 .word 0x41807200 @ ARM720T (writethrough) 1077 .word 0xffffff00 1078 W(b) __armv4_mmu_cache_on 1079 W(b) __armv4_mmu_cache_off 1080 mov pc, lr 1081 THUMB( nop ) 1082 1083 .word 0x41007400 @ ARM74x 1084 .word 0xff00ff00 1085 W(b) __armv3_mpu_cache_on 1086 W(b) __armv3_mpu_cache_off 1087 W(b) __armv3_mpu_cache_flush 1088 1089 .word 0x41009400 @ ARM94x 1090 .word 0xff00ff00 1091 W(b) __armv4_mpu_cache_on 1092 W(b) __armv4_mpu_cache_off 1093 W(b) __armv4_mpu_cache_flush 1094 1095 .word 0x41069260 @ ARM926EJ-S (v5TEJ) 1096 .word 0xff0ffff0 1097 W(b) __arm926ejs_mmu_cache_on 1098 W(b) __armv4_mmu_cache_off 1099 W(b) __armv5tej_mmu_cache_flush 1100 1101 .word 0x00007000 @ ARM7 IDs 1102 .word 0x0000f000 1103 mov pc, lr 1104 THUMB( nop ) 1105 mov pc, lr 1106 THUMB( nop ) 1107 mov pc, lr 1108 THUMB( nop ) 1109 1110 @ Everything from here on will be the new ID system. 1111 1112 .word 0x4401a100 @ sa110 / sa1100 1113 .word 0xffffffe0 1114 W(b) __armv4_mmu_cache_on 1115 W(b) __armv4_mmu_cache_off 1116 W(b) __armv4_mmu_cache_flush 1117 1118 .word 0x6901b110 @ sa1110 1119 .word 0xfffffff0 1120 W(b) __armv4_mmu_cache_on 1121 W(b) __armv4_mmu_cache_off 1122 W(b) __armv4_mmu_cache_flush 1123 1124 .word 0x56056900 1125 .word 0xffffff00 @ PXA9xx 1126 W(b) __armv4_mmu_cache_on 1127 W(b) __armv4_mmu_cache_off 1128 W(b) __armv4_mmu_cache_flush 1129 1130 .word 0x56158000 @ PXA168 1131 .word 0xfffff000 1132 W(b) __armv4_mmu_cache_on 1133 W(b) __armv4_mmu_cache_off 1134 W(b) __armv5tej_mmu_cache_flush 1135 1136 .word 0x56050000 @ Feroceon 1137 .word 0xff0f0000 1138 W(b) __armv4_mmu_cache_on 1139 W(b) __armv4_mmu_cache_off 1140 W(b) __armv5tej_mmu_cache_flush 1141 1142#ifdef CONFIG_CPU_FEROCEON_OLD_ID 1143 /* this conflicts with the standard ARMv5TE entry */ 1144 .long 0x41009260 @ Old Feroceon 1145 .long 0xff00fff0 1146 b __armv4_mmu_cache_on 1147 b __armv4_mmu_cache_off 1148 b __armv5tej_mmu_cache_flush 1149#endif 1150 1151 .word 0x66015261 @ FA526 1152 .word 0xff01fff1 1153 W(b) __fa526_cache_on 1154 W(b) __armv4_mmu_cache_off 1155 W(b) __fa526_cache_flush 1156 1157 @ These match on the architecture ID 1158 1159 .word 0x00020000 @ ARMv4T 1160 .word 0x000f0000 1161 W(b) __armv4_mmu_cache_on 1162 W(b) __armv4_mmu_cache_off 1163 W(b) __armv4_mmu_cache_flush 1164 1165 .word 0x00050000 @ ARMv5TE 1166 .word 0x000f0000 1167 W(b) __armv4_mmu_cache_on 1168 W(b) __armv4_mmu_cache_off 1169 W(b) __armv4_mmu_cache_flush 1170 1171 .word 0x00060000 @ ARMv5TEJ 1172 .word 0x000f0000 1173 W(b) __armv4_mmu_cache_on 1174 W(b) __armv4_mmu_cache_off 1175 W(b) __armv5tej_mmu_cache_flush 1176 1177 .word 0x0007b000 @ ARMv6 1178 .word 0x000ff000 1179 W(b) __armv6_mmu_cache_on 1180 W(b) __armv4_mmu_cache_off 1181 W(b) __armv6_mmu_cache_flush 1182 1183 .word 0x000f0000 @ new CPU Id 1184 .word 0x000f0000 1185 W(b) __armv7_mmu_cache_on 1186 W(b) __armv7_mmu_cache_off 1187 W(b) __armv7_mmu_cache_flush 1188 1189 .word 0 @ unrecognised type 1190 .word 0 1191 mov pc, lr 1192 THUMB( nop ) 1193 mov pc, lr 1194 THUMB( nop ) 1195 mov pc, lr 1196 THUMB( nop ) 1197 1198 .size proc_types, . - proc_types 1199 1200 /* 1201 * If you get a "non-constant expression in ".if" statement" 1202 * error from the assembler on this line, check that you have 1203 * not accidentally written a "b" instruction where you should 1204 * have written W(b). 1205 */ 1206 .if (. - proc_types) % PROC_ENTRY_SIZE != 0 1207 .error "The size of one or more proc_types entries is wrong." 1208 .endif 1209 1210/* 1211 * Turn off the Cache and MMU. ARMv3 does not support 1212 * reading the control register, but ARMv4 does. 1213 * 1214 * On exit, 1215 * r0, r1, r2, r3, r9, r12 corrupted 1216 * This routine must preserve: 1217 * r4, r7, r8 1218 */ 1219 .align 5 1220cache_off: mov r3, #12 @ cache_off function 1221 b call_cache_fn 1222 1223__armv4_mpu_cache_off: 1224 mrc p15, 0, r0, c1, c0 1225 bic r0, r0, #0x000d 1226 mcr p15, 0, r0, c1, c0 @ turn MPU and cache off 1227 mov r0, #0 1228 mcr p15, 0, r0, c7, c10, 4 @ drain write buffer 1229 mcr p15, 0, r0, c7, c6, 0 @ flush D-Cache 1230 mcr p15, 0, r0, c7, c5, 0 @ flush I-Cache 1231 mov pc, lr 1232 1233__armv3_mpu_cache_off: 1234 mrc p15, 0, r0, c1, c0 1235 bic r0, r0, #0x000d 1236 mcr p15, 0, r0, c1, c0, 0 @ turn MPU and cache off 1237 mov r0, #0 1238 mcr p15, 0, r0, c7, c0, 0 @ invalidate whole cache v3 1239 mov pc, lr 1240 1241__armv4_mmu_cache_off: 1242#ifdef CONFIG_MMU 1243 mrc p15, 0, r0, c1, c0 1244 bic r0, r0, #0x000d 1245 mcr p15, 0, r0, c1, c0 @ turn MMU and cache off 1246 mov r0, #0 1247 mcr p15, 0, r0, c7, c7 @ invalidate whole cache v4 1248 mcr p15, 0, r0, c8, c7 @ invalidate whole TLB v4 1249#endif 1250 mov pc, lr 1251 1252__armv7_mmu_cache_off: 1253 mrc p15, 0, r0, c1, c0 1254#ifdef CONFIG_MMU 1255 bic r0, r0, #0x0005 1256#else 1257 bic r0, r0, #0x0004 1258#endif 1259 mcr p15, 0, r0, c1, c0 @ turn MMU and cache off 1260 mov r0, #0 1261#ifdef CONFIG_MMU 1262 mcr p15, 0, r0, c8, c7, 0 @ invalidate whole TLB 1263#endif 1264 mcr p15, 0, r0, c7, c5, 6 @ invalidate BTC 1265 mcr p15, 0, r0, c7, c10, 4 @ DSB 1266 mcr p15, 0, r0, c7, c5, 4 @ ISB 1267 mov pc, lr 1268 1269/* 1270 * Clean and flush the cache to maintain consistency. 1271 * 1272 * On entry, 1273 * r0 = start address 1274 * r1 = end address (exclusive) 1275 * On exit, 1276 * r1, r2, r3, r9, r10, r11, r12 corrupted 1277 * This routine must preserve: 1278 * r4, r6, r7, r8 1279 */ 1280 .align 5 1281cache_clean_flush: 1282 mov r3, #16 1283 mov r11, r1 1284 b call_cache_fn 1285 1286__armv4_mpu_cache_flush: 1287 tst r4, #1 1288 movne pc, lr 1289 mov r2, #1 1290 mov r3, #0 1291 mcr p15, 0, ip, c7, c6, 0 @ invalidate D cache 1292 mov r1, #7 << 5 @ 8 segments 12931: orr r3, r1, #63 << 26 @ 64 entries 12942: mcr p15, 0, r3, c7, c14, 2 @ clean & invalidate D index 1295 subs r3, r3, #1 << 26 1296 bcs 2b @ entries 63 to 0 1297 subs r1, r1, #1 << 5 1298 bcs 1b @ segments 7 to 0 1299 1300 teq r2, #0 1301 mcrne p15, 0, ip, c7, c5, 0 @ invalidate I cache 1302 mcr p15, 0, ip, c7, c10, 4 @ drain WB 1303 mov pc, lr 1304 1305__fa526_cache_flush: 1306 tst r4, #1 1307 movne pc, lr 1308 mov r1, #0 1309 mcr p15, 0, r1, c7, c14, 0 @ clean and invalidate D cache 1310 mcr p15, 0, r1, c7, c5, 0 @ flush I cache 1311 mcr p15, 0, r1, c7, c10, 4 @ drain WB 1312 mov pc, lr 1313 1314__armv6_mmu_cache_flush: 1315 mov r1, #0 1316 tst r4, #1 1317 mcreq p15, 0, r1, c7, c14, 0 @ clean+invalidate D 1318 mcr p15, 0, r1, c7, c5, 0 @ invalidate I+BTB 1319 mcreq p15, 0, r1, c7, c15, 0 @ clean+invalidate unified 1320 mcr p15, 0, r1, c7, c10, 4 @ drain WB 1321 mov pc, lr 1322 1323__armv7_mmu_cache_flush: 1324 enable_cp15_barriers r10 1325 tst r4, #1 1326 bne iflush 1327 mrc p15, 0, r10, c0, c1, 5 @ read ID_MMFR1 1328 tst r10, #0xf << 16 @ hierarchical cache (ARMv7) 1329 mov r10, #0 1330 beq hierarchical 1331 mcr p15, 0, r10, c7, c14, 0 @ clean+invalidate D 1332 b iflush 1333hierarchical: 1334 dcache_line_size r1, r2 @ r1 := dcache min line size 1335 sub r2, r1, #1 @ r2 := line size mask 1336 bic r0, r0, r2 @ round down start to line size 1337 sub r11, r11, #1 @ end address is exclusive 1338 bic r11, r11, r2 @ round down end to line size 13390: cmp r0, r11 @ finished? 1340 bgt iflush 1341 mcr p15, 0, r0, c7, c14, 1 @ Dcache clean/invalidate by VA 1342 add r0, r0, r1 1343 b 0b 1344iflush: 1345 mcr p15, 0, r10, c7, c10, 4 @ DSB 1346 mcr p15, 0, r10, c7, c5, 0 @ invalidate I+BTB 1347 mcr p15, 0, r10, c7, c10, 4 @ DSB 1348 mcr p15, 0, r10, c7, c5, 4 @ ISB 1349 mov pc, lr 1350 1351__armv5tej_mmu_cache_flush: 1352 tst r4, #1 1353 movne pc, lr 13541: mrc p15, 0, APSR_nzcv, c7, c14, 3 @ test,clean,invalidate D cache 1355 bne 1b 1356 mcr p15, 0, r0, c7, c5, 0 @ flush I cache 1357 mcr p15, 0, r0, c7, c10, 4 @ drain WB 1358 mov pc, lr 1359 1360__armv4_mmu_cache_flush: 1361 tst r4, #1 1362 movne pc, lr 1363 mov r2, #64*1024 @ default: 32K dcache size (*2) 1364 mov r11, #32 @ default: 32 byte line size 1365 mrc p15, 0, r3, c0, c0, 1 @ read cache type 1366 teq r3, r9 @ cache ID register present? 1367 beq no_cache_id 1368 mov r1, r3, lsr #18 1369 and r1, r1, #7 1370 mov r2, #1024 1371 mov r2, r2, lsl r1 @ base dcache size *2 1372 tst r3, #1 << 14 @ test M bit 1373 addne r2, r2, r2, lsr #1 @ +1/2 size if M == 1 1374 mov r3, r3, lsr #12 1375 and r3, r3, #3 1376 mov r11, #8 1377 mov r11, r11, lsl r3 @ cache line size in bytes 1378no_cache_id: 1379 mov r1, pc 1380 bic r1, r1, #63 @ align to longest cache line 1381 add r2, r1, r2 13821: 1383 ARM( ldr r3, [r1], r11 ) @ s/w flush D cache 1384 THUMB( ldr r3, [r1] ) @ s/w flush D cache 1385 THUMB( add r1, r1, r11 ) 1386 teq r1, r2 1387 bne 1b 1388 1389 mcr p15, 0, r1, c7, c5, 0 @ flush I cache 1390 mcr p15, 0, r1, c7, c6, 0 @ flush D cache 1391 mcr p15, 0, r1, c7, c10, 4 @ drain WB 1392 mov pc, lr 1393 1394__armv3_mmu_cache_flush: 1395__armv3_mpu_cache_flush: 1396 tst r4, #1 1397 movne pc, lr 1398 mov r1, #0 1399 mcr p15, 0, r1, c7, c0, 0 @ invalidate whole cache v3 1400 mov pc, lr 1401 1402/* 1403 * Various debugging routines for printing hex characters and 1404 * memory, which again must be relocatable. 1405 */ 1406#ifdef DEBUG 1407 .align 2 1408 .type phexbuf,#object 1409phexbuf: .space 12 1410 .size phexbuf, . - phexbuf 1411 1412@ phex corrupts {r0, r1, r2, r3} 1413phex: adr r3, phexbuf 1414 mov r2, #0 1415 strb r2, [r3, r1] 14161: subs r1, r1, #1 1417 movmi r0, r3 1418 bmi puts 1419 and r2, r0, #15 1420 mov r0, r0, lsr #4 1421 cmp r2, #10 1422 addge r2, r2, #7 1423 add r2, r2, #'0' 1424 strb r2, [r3, r1] 1425 b 1b 1426 1427@ puts corrupts {r0, r1, r2, r3} 1428puts: loadsp r3, r2, r1 14291: ldrb r2, [r0], #1 1430 teq r2, #0 1431 moveq pc, lr 14322: writeb r2, r3, r1 1433 mov r1, #0x00020000 14343: subs r1, r1, #1 1435 bne 3b 1436 teq r2, #'\n' 1437 moveq r2, #'\r' 1438 beq 2b 1439 teq r0, #0 1440 bne 1b 1441 mov pc, lr 1442@ putc corrupts {r0, r1, r2, r3} 1443putc: 1444 mov r2, r0 1445 loadsp r3, r1, r0 1446 mov r0, #0 1447 b 2b 1448 1449@ memdump corrupts {r0, r1, r2, r3, r10, r11, r12, lr} 1450memdump: mov r12, r0 1451 mov r10, lr 1452 mov r11, #0 14532: mov r0, r11, lsl #2 1454 add r0, r0, r12 1455 mov r1, #8 1456 bl phex 1457 mov r0, #':' 1458 bl putc 14591: mov r0, #' ' 1460 bl putc 1461 ldr r0, [r12, r11, lsl #2] 1462 mov r1, #8 1463 bl phex 1464 and r0, r11, #7 1465 teq r0, #3 1466 moveq r0, #' ' 1467 bleq putc 1468 and r0, r11, #7 1469 add r11, r11, #1 1470 teq r0, #7 1471 bne 1b 1472 mov r0, #'\n' 1473 bl putc 1474 cmp r11, #64 1475 blt 2b 1476 mov pc, r10 1477#endif 1478 1479 .ltorg 1480 1481#ifdef CONFIG_ARM_VIRT_EXT 1482.align 5 1483__hyp_reentry_vectors: 1484 W(b) . @ reset 1485 W(b) . @ undef 1486#ifdef CONFIG_EFI_STUB 1487 W(b) __enter_kernel_from_hyp @ hvc from HYP 1488#else 1489 W(b) . @ svc 1490#endif 1491 W(b) . @ pabort 1492 W(b) . @ dabort 1493 W(b) __enter_kernel @ hyp 1494 W(b) . @ irq 1495 W(b) . @ fiq 1496#endif /* CONFIG_ARM_VIRT_EXT */ 1497 1498__enter_kernel: 1499 mov r0, #0 @ must be 0 1500 mov r1, r7 @ restore architecture number 1501 mov r2, r8 @ restore atags pointer 1502#ifdef CONFIG_RANDOMIZE_BASE 1503 ldr r3, __kaslr_offset 1504 add r4, r4, #4 @ skip first instruction 1505#endif 1506 ARM( mov pc, r4 ) @ call kernel 1507 M_CLASS( add r4, r4, #1 ) @ enter in Thumb mode for M class 1508 THUMB( bx r4 ) @ entry point is always ARM for A/R classes 1509 1510#ifdef CONFIG_RANDOMIZE_BASE 1511 /* 1512 * Minimal implementation of CRC-16 that does not use a 1513 * lookup table and uses 32-bit wide loads, so it still 1514 * performs reasonably well with the D-cache off. Equivalent 1515 * to lib/crc16.c for input sizes that are 4 byte multiples. 1516 */ 1517ENTRY(__crc16) 1518 push {r4, lr} 1519 ldr r3, =0xa001 @ CRC-16 polynomial 15200: subs r2, r2, #4 1521 popmi {r4, pc} 1522 ldr r4, [r1], #4 1523#ifdef __ARMEB__ 1524 eor ip, r4, r4, ror #16 @ endian swap 1525 bic ip, ip, #0x00ff0000 1526 mov r4, r4, ror #8 1527 eor r4, r4, ip, lsr #8 1528#endif 1529 eor r0, r0, r4 1530 .rept 32 1531 lsrs r0, r0, #1 1532 eorcs r0, r0, r3 1533 .endr 1534 b 0b 1535ENDPROC(__crc16) 1536 1537 .align 2 1538__kaslr_seed: .long 0 1539__kaslr_offset: .long 0 1540#endif 1541 1542reloc_code_end: 1543 1544#ifdef CONFIG_EFI_STUB 1545__enter_kernel_from_hyp: 1546 mrc p15, 4, r0, c1, c0, 0 @ read HSCTLR 1547 bic r0, r0, #0x5 @ disable MMU and caches 1548 mcr p15, 4, r0, c1, c0, 0 @ write HSCTLR 1549 isb 1550 b __enter_kernel 1551 1552ENTRY(efi_enter_kernel) 1553 mov r4, r0 @ preserve image base 1554 mov r8, r1 @ preserve DT pointer 1555 1556 adr_l r0, call_cache_fn 1557 adr r1, 0f @ clean the region of code we 1558 bl cache_clean_flush @ may run with the MMU off 1559 1560#ifdef CONFIG_ARM_VIRT_EXT 1561 @ 1562 @ The EFI spec does not support booting on ARM in HYP mode, 1563 @ since it mandates that the MMU and caches are on, with all 1564 @ 32-bit addressable DRAM mapped 1:1 using short descriptors. 1565 @ 1566 @ While the EDK2 reference implementation adheres to this, 1567 @ U-Boot might decide to enter the EFI stub in HYP mode 1568 @ anyway, with the MMU and caches either on or off. 1569 @ 1570 mrs r0, cpsr @ get the current mode 1571 msr spsr_cxsf, r0 @ record boot mode 1572 and r0, r0, #MODE_MASK @ are we running in HYP mode? 1573 cmp r0, #HYP_MODE 1574 bne .Lefi_svc 1575 1576 mrc p15, 4, r1, c1, c0, 0 @ read HSCTLR 1577 tst r1, #0x1 @ MMU enabled at HYP? 1578 beq 1f 1579 1580 @ 1581 @ When running in HYP mode with the caches on, we're better 1582 @ off just carrying on using the cached 1:1 mapping that the 1583 @ firmware provided. Set up the HYP vectors so HVC instructions 1584 @ issued from HYP mode take us to the correct handler code. We 1585 @ will disable the MMU before jumping to the kernel proper. 1586 @ 1587 ARM( bic r1, r1, #(1 << 30) ) @ clear HSCTLR.TE 1588 THUMB( orr r1, r1, #(1 << 30) ) @ set HSCTLR.TE 1589 mcr p15, 4, r1, c1, c0, 0 1590 adr r0, __hyp_reentry_vectors 1591 mcr p15, 4, r0, c12, c0, 0 @ set HYP vector base (HVBAR) 1592 isb 1593 b .Lefi_hyp 1594 1595 @ 1596 @ When running in HYP mode with the caches off, we need to drop 1597 @ into SVC mode now, and let the decompressor set up its cached 1598 @ 1:1 mapping as usual. 1599 @ 16001: mov r9, r4 @ preserve image base 1601 bl __hyp_stub_install @ install HYP stub vectors 1602 safe_svcmode_maskall r1 @ drop to SVC mode 1603 msr spsr_cxsf, r0 @ record boot mode 1604 orr r4, r9, #1 @ restore image base and set LSB 1605 b .Lefi_hyp 1606.Lefi_svc: 1607#endif 1608 mrc p15, 0, r0, c1, c0, 0 @ read SCTLR 1609 tst r0, #0x1 @ MMU enabled? 1610 orreq r4, r4, #1 @ set LSB if not 1611 1612.Lefi_hyp: 1613 mov r0, r8 @ DT start 1614 add r1, r8, r2 @ DT end 1615 bl cache_clean_flush 1616 1617 adr r0, 0f @ switch to our stack 1618 ldr sp, [r0] 1619 add sp, sp, r0 1620 1621 mov r5, #0 @ appended DTB size 1622 mov r7, #0xFFFFFFFF @ machine ID 1623 b wont_overwrite 1624ENDPROC(efi_enter_kernel) 16250: .long .L_user_stack_end - . 1626#endif 1627 1628 .align 1629 .section ".stack", "aw", %nobits 1630.L_user_stack: .space 4096 1631.L_user_stack_end: 1632