ARM cache

grub-core/kern/arm/cache.S

@@ -37,20 +37,16 @@
  * Simple cache maintenance functions
  */
 
-dlinesz_addr:
-	.long EXT_C(grub_arch_cache_dlinesz)
-ilinesz_addr:
-	.long EXT_C(grub_arch_cache_ilinesz)
-
 @ r0 - *beg (inclusive)
 @ r1 - *end (exclusive)
-clean_dcache_range:
+@void grub_arm_clean_dcache_range (grub_addr_t start, grub_addr_t end, grub_addr_t dlinesz)
+#ifdef ARMV6
+FUNCTION(grub_arm_clean_dcache_range_armv6)
+#else
+FUNCTION(grub_arm_clean_dcache_range_armv7)
+#endif
+	DSB
 	@ Clean data cache for range to point-of-unification
-	ldr	r2, dlinesz_addr
-	ldr	r2, [r2]
-	sub	r3, r2, #1		@ align "beg" to start of line
-	mvn	r3, r3
-	and	r0, r0, r3
 1:	cmp	r0, r1
 	bge	2f
 #ifdef ARMV6
@@ -65,13 +61,12 @@ clean_dcache_range:
 
 @ r0 - *beg (inclusive)
 @ r1 - *end (exclusive)
-invalidate_icache_range:
+#ifdef ARMV6
+FUNCTION(grub_arm_invalidate_icache_range_armv6)
+#else
+FUNCTION(grub_arm_invalidate_icache_range_armv7)
+#endif
 	@ Invalidate instruction cache for range to point-of-unification
-	ldr	r2, ilinesz_addr
-	ldr	r2, [r2]
-	sub	r3, r2, #1		@ align "beg" to start of line
-	mvn	r3, r3
-	and	r0, r0, r3
 1:	cmp	r0, r1
 	bge	2f
 	mcr	p15, 0, r0, c7, c5, 1	@ ICIMVAU
@@ -83,21 +78,6 @@ invalidate_icache_range:
 	ISB
 	bx	lr
 
-@void grub_arch_sync_caches (void *address, grub_size_t len)
-#ifdef ARMV6
-FUNCTION(grub_arch_sync_caches_armv6)
-#else
-FUNCTION(grub_arch_sync_caches_armv7)
-#endif
-	DSB
-	add	r1, r0, r1
-	push	{r0-r2, lr}
-	bl	clean_dcache_range
-	pop	{r0, r1}
-	bl	invalidate_icache_range
-	pop	{r2, lr}
-	bx	lr
-
 #ifdef ARMV6
 FUNCTION(grub_arm_disable_caches_mmu_armv6)
 #else

grub-core/kern/arm/cache.c

@@ -1,6 +1,11 @@
 #include <grub/dl.h>
 #include <grub/cache.h>
 #include <grub/arm/system.h>
+#ifdef GRUB_MACHINE_UBOOT
+#include <grub/uboot/uboot.h>
+#include <grub/uboot/api_public.h>
+#include <grub/mm.h>
+#endif
 
 /* This is only about cache architecture. It doesn't imply
    the CPU architecture.  */
@@ -13,12 +18,21 @@ static enum
     ARCH_ARMV7
   } type = ARCH_UNKNOWN;
 
-grub_uint32_t grub_arch_cache_dlinesz;
+static int is_v6_mmu;
-grub_uint32_t grub_arch_cache_ilinesz;
+
+static grub_uint32_t grub_arch_cache_dlinesz;
+static grub_uint32_t grub_arch_cache_ilinesz;
+static grub_uint32_t grub_arch_cache_max_linesz;
 
 /* Prototypes for asm functions.  */
-void grub_arch_sync_caches_armv6 (void *address, grub_size_t len);
+void grub_arm_clean_dcache_range_armv6 (grub_addr_t start, grub_addr_t end,
-void grub_arch_sync_caches_armv7 (void *address, grub_size_t len);
+					grub_addr_t dlinesz);
+void grub_arm_clean_dcache_range_armv7 (grub_addr_t start, grub_addr_t end,
+					grub_addr_t dlinesz);
+void grub_arm_invalidate_icache_range_armv6 (grub_addr_t start, grub_addr_t end,
+					     grub_addr_t dlinesz);
+void grub_arm_invalidate_icache_range_armv7 (grub_addr_t start, grub_addr_t end,
+					     grub_addr_t dlinesz);
 void grub_arm_disable_caches_mmu_armv6 (void);
 void grub_arm_disable_caches_mmu_armv7 (void);
 grub_uint32_t grub_arm_main_id (void);
@@ -38,8 +52,11 @@ probe_caches (void)
     case 0x4:
     case 0x5:
     case 0x6:
+      is_v6_mmu = 0;
+      break;
     case 0x7:
     case 0xf:
+      is_v6_mmu = 1;
       break;
     default:
       grub_fatal ("Unsupported ARM ID 0x%x", main_id);
@@ -82,20 +99,148 @@ probe_caches (void)
     default:
       grub_fatal ("Unsupported cache type 0x%x", cache_type);
     }
+  if (grub_arch_cache_dlinesz > grub_arch_cache_ilinesz)
+    grub_arch_cache_max_linesz = grub_arch_cache_dlinesz;
+  else
+    grub_arch_cache_max_linesz = grub_arch_cache_ilinesz;
 }
 
+#ifdef GRUB_MACHINE_UBOOT
+
+static void subdivide (grub_uint32_t *table, grub_uint32_t *subtable,
+		       grub_uint32_t addr)
+{
+  grub_uint32_t j;
+  addr = addr >> 20 << 20;
+  table[addr >> 20] = (grub_addr_t) subtable | 1;
+  for (j = 0; j < 256; j++)
+    subtable[j] = addr | (j << 12)
+      | (3 << 4) | (3 << 6) | (3 << 8) | (3 << 10)
+      | (0 << 3) | (1 << 2) | 2;
+}
+
+void
+grub_arm_enable_caches_mmu (void)
+{
+  grub_uint32_t *table;
+  grub_uint32_t i;
+  grub_uint32_t border_crossing = 0;
+  grub_uint32_t *subtable;
+  struct sys_info *si = grub_uboot_get_sys_info ();
+
+  if (!si || (si->mr_no == 0))
+    {
+      grub_printf ("couldn't get memory map, not enabling caches");
+      grub_errno = GRUB_ERR_NONE;
+      return;
+    }
+
+  if (type == ARCH_UNKNOWN)
+    probe_caches ();
+
+  for (i = 0; (signed) i < si->mr_no; i++)
+    {
+      if (si->mr[i].start & ((1 << 20) - 1))
+	border_crossing++;
+      if ((si->mr[i].start + si->mr[i].size) & ((1 << 20) - 1))
+	border_crossing++;
+    }
+
+  grub_printf ("%d crossers\n", border_crossing);
+
+  table = grub_memalign (1 << 14, (1 << 14) + (border_crossing << 10));
+  if (!table)
+    {
+      grub_printf ("couldn't allocate place for MMU table, not enabling caches");
+      grub_errno = GRUB_ERR_NONE;
+      return;
+    }
+
+  subtable = table + (1 << 12);
+  /* Map all unknown as device.  */
+  for (i = 0; i < (1 << 12); i++)
+    table[i] = (i << 20) | (3 << 10) | (0 << 3) | (1 << 2) | 2;
+  /*
+    Device: TEX= 0, C=0, B=1
+    normal: TEX= 0, C=1, B=1
+    AP = 3
+    IMP = 0
+    Domain = 0
+*/
+
+  for (i = 0; (signed) i < si->mr_no; i++)
+    {
+      if (si->mr[i].start & ((1 << 20) - 1))
+	{
+	  subdivide (table, subtable, si->mr[i].start);
+	  subtable += (1 << 8);
+	}
+      if ((si->mr[i].start + si->mr[i].size) & ((1 << 20) - 1))
+	{
+	  subdivide (table, subtable, si->mr[i].start + si->mr[i].size);
+	  subtable += (1 << 8);
+	}
+    }
+
+  for (i = 0; (signed) i < si->mr_no; i++)
+    if ((si->mr[i].flags & MR_ATTR_MASK) == MR_ATTR_DRAM
+	|| (si->mr[i].flags & MR_ATTR_MASK) == MR_ATTR_SRAM
+	|| (si->mr[i].flags & MR_ATTR_MASK) == MR_ATTR_FLASH)
+      {
+	grub_uint32_t cur, end;
+	cur = si->mr[i].start;
+	end = si->mr[i].start + si->mr[i].size;
+	while (cur < end)
+	  {
+	    grub_uint32_t *st;
+	    if ((table[cur >> 20] & 3) == 2)
+	      {
+		cur = cur >> 20 << 20;
+		table[cur >> 20] = cur | (3 << 10) | (1 << 3) | (1 << 2) | 2;
+		cur += (1 << 20);
+		continue;
+	      }
+	    cur = cur >> 12 << 12;
+	    st = (grub_uint32_t *) (table[cur >> 20] & ~0x3ff);
+	    st[(cur >> 12) & 0xff] = cur | (3 << 4) | (3 << 6)
+	      | (3 << 8) | (3 << 10)
+	      | (1 << 3) | (1 << 2) | 2;
+	    cur += (1 << 12);
+	  }
+      }
+
+  grub_printf ("MMU tables generated\n");
+  if (is_v6_mmu)
+    grub_arm_clear_mmu_v6 ();
+
+  grub_printf ("enabling MMU\n");
+  grub_arm_enable_mmu (table);
+  grub_printf ("MMU enabled\n");
+}
+
+#endif
+
 void
 grub_arch_sync_caches (void *address, grub_size_t len)
 {
+  grub_addr_t start = (grub_addr_t) address;
+  grub_addr_t end = start + len;
+
   if (type == ARCH_UNKNOWN)
     probe_caches ();
+  start = ALIGN_DOWN (start, grub_arch_cache_max_linesz);
+  end = ALIGN_UP (end, grub_arch_cache_max_linesz);
   switch (type)
     {
     case ARCH_ARMV6:
-      grub_arch_sync_caches_armv6 (address, len);
+      grub_arm_clean_dcache_range_armv6 (start, end, grub_arch_cache_dlinesz);
+      grub_arm_invalidate_icache_range_armv6 (start, end,
+					      grub_arch_cache_ilinesz);
       break;
     case ARCH_ARMV7:
-      grub_arch_sync_caches_armv7 (address, len);
+      grub_arm_clean_dcache_range_armv7 (start, end, grub_arch_cache_dlinesz);
+      grub_arm_invalidate_icache_range_armv7 (start, end,
+					      grub_arch_cache_ilinesz);
       break;
       /* Nothing to do.  */
     case ARCH_ARMV5_WRITE_THROUGH:

grub-core/kern/arm/cache_armv6.S

@@ -43,10 +43,30 @@ FUNCTION(grub_arm_cache_type)
 	mrc 	p15, 0, r0, c0, c0, 1
 	bx lr
 
-FUNCTION(grub_arm_cache_enable)
+FUNCTION(grub_arm_clear_mmu_v6)
+	mov	r0, #0
+	mcr	p15, 0, r0, c2, c0, 2
+	mvn	r0, #0
+	mcr	p15, 0, r0, c3, c0, 0
+	bx	lr
+
+FUNCTION(grub_arm_enable_mmu)
+	mcr	p15, 0, r0, c2, c0, 0
+
+	mrc	p15, 0, r0, c1, c0, 0
+	bic	r0, r0, #(1 << 23)
+	mcr	p15, 0, r0, c1, c0, 0
+
 	mrc	p15, 0, r0, c1, c0, 0
-	orr	r0, r0, #(1 << 12)
-	orr	r0, r0, #(1 << 2)
 	orr	r0, r0, #(1 << 0)
 	mcr	p15, 0, r0, c1, c0, 0
+
+	mrc	p15, 0, r0, c1, c0, 0
+	orr	r0, r0, #(1 << 2)
+	mcr	p15, 0, r0, c1, c0, 0
+
+	mrc	p15, 0, r0, c1, c0, 0
+	orr	r0, r0, #(1 << 12)
+	mcr	p15, 0, r0, c1, c0, 0
+
 	bx	lr

grub-core/kern/uboot/init.c

@@ -31,6 +31,7 @@
 #include <grub/uboot/uboot.h>
 #include <grub/uboot/api_public.h>
 #include <grub/cpu/system.h>
+#include <grub/cache.h>
 
 extern char __bss_start[];
 extern char _end[];
@@ -83,8 +84,6 @@ rpi_timer_ms (void)
 }
 #endif
 
-void grub_arm_cache_enable (void);
-
 void
 grub_machine_init (void)
 {
@@ -103,16 +102,17 @@ grub_machine_init (void)
       grub_uboot_puts ("invalid U-Boot API version\n");
     }
 
-#ifdef __arm__
-  grub_arm_cache_enable ();
-#endif
-
   /* Initialize the console so that GRUB can display messages.  */
   grub_console_init_early ();
 
   /* Enumerate memory and initialize the memory management system. */
   grub_uboot_mm_init ();
 
+  /* Shold be earlier but it needs memalign.  */
+#ifdef __arm__
+  grub_arm_enable_caches_mmu ();
+#endif
+
   grub_dprintf ("init", "__bss_start: %p\n", __bss_start);
   grub_dprintf ("init", "_end: %p\n", _end);
   grub_dprintf ("init", "grub_modbase: %p\n", (void *) grub_modbase);

include/grub/arm/system.h

@@ -1,6 +1,8 @@
 #ifndef GRUB_SYSTEM_CPU_HEADER
 #define GRUB_SYSTEM_CPU_HEADER
 
+#include <grub/types.h>
+
 enum
   {
     GRUB_ARM_MACHINE_TYPE_RASPBERRY_PI = 3138,
@@ -8,6 +10,9 @@ enum
   };
 
 void grub_arm_disable_caches_mmu (void);
+void grub_arm_enable_caches_mmu (void);
+void grub_arm_enable_mmu (grub_uint32_t *mmu_tables);
+void grub_arm_clear_mmu_v6 (void);
 
 #endif /* ! GRUB_SYSTEM_CPU_HEADER */
 

include/grub/cache.h

@@ -45,4 +45,9 @@ grub_arch_sync_dma_caches (void *address __attribute__ ((unused)),
 #endif
 #endif
 
+#ifdef __arm__
+void
+grub_arm_cache_enable (void);
+#endif
+
 #endif /* ! GRUB_CACHE_HEADER */