[RFC][patch] Make DCACHE_LINE runtime-settable

[ppluzhnikov@google.com (Paul Pluzhnikov)]

Greetings,

Another followup to my earlier "slow on high-latency links" message:
http://sourceware.org/ml/gdb-patches/2011-07/msg00391.html ...

With this change:

2009-08-31  Jacob Potter  <jdpotter@google.com>
            Doug Evans  <dje@google.com>

        Implement TARGET_OBJECT_STACK_MEMORY.
        * NEWS: Add note on new "set stack-cache" option.
        * corefile.c (read_stack): New function.
        * dcache.c (dcache_struct): New member ptid.
        ...

gdb no longer reads stack 4 bytes at a time (good). Instead it reads
LINE_SIZE (== 64) bytes at a time.

With typical x86_64 frame sizes, I see about 1 packet per frame level.

For a 200ms round-trip connection, executing "where" and getting back 30
frames translates into 6 seconds of wall time, and "thread apply all where"
into 1 to 10 to 100 minutes (depending on how many threads there are,
with 100 threads being common and 1000 not being exceptional).

Attached patch allows the user to e.g. "set dcache-line-size 4096" and
drastically reduce the number of round trips.

I've also added "set dcache-size" just for good measure.

Not sure whether this needs documentation updates.

Thanks,

--
Paul Pluzhnikov

2011-07-22  Paul Pluzhnikov  <ppluzhnikov@google.com>

	* dcache.c (dcache_size): New variable
	(dcache_line_size): Likewise.
	(struct dcache_block): Make it expandable.
	(struct dcache_struct): New field.
	(dcache_invalidate): Discard freelist upon dcache_line_size changes.
	(dcache_hit, dcache_alloc, dcache_peek_byte): Adjust.
	(dcache_poke_byte, dcache_print_line): Adjust.
	(set_dcache_size, set_dcache_line_size): New functions.
	(_initialize_dcache): Add new commands.


Index: dcache.c
===================================================================
RCS file: /cvs/src/src/gdb/dcache.c,v
retrieving revision 1.45
diff -u -p -r1.45 dcache.c
--- dcache.c	7 Jan 2011 19:36:15 -0000	1.45
+++ dcache.c	22 Jul 2011 21:59:07 -0000
@@ -71,20 +71,21 @@
 
 /* The maximum number of lines stored.  The total size of the cache is
    equal to DCACHE_SIZE times LINE_SIZE.  */
-#define DCACHE_SIZE 4096
+#define DCACHE_DEFAULT_SIZE 4096
+static unsigned dcache_size = DCACHE_DEFAULT_SIZE;
 
-/* The size of a cache line.  Smaller values reduce the time taken to
+/* The default size of a cache line.  Smaller values reduce the time taken to
    read a single byte and make the cache more granular, but increase
    overhead and reduce the effectiveness of the cache as a prefetcher.  */
-#define LINE_SIZE_POWER 6
-#define LINE_SIZE (1 << LINE_SIZE_POWER)
+#define DCACHE_DEFAULT_LINE_SIZE 64
+static unsigned dcache_line_size = DCACHE_DEFAULT_LINE_SIZE;
 
 /* Each cache block holds LINE_SIZE bytes of data
    starting at a multiple-of-LINE_SIZE address.  */
 
-#define LINE_SIZE_MASK  ((LINE_SIZE - 1))
-#define XFORM(x) 	((x) & LINE_SIZE_MASK)
-#define MASK(x)         ((x) & ~LINE_SIZE_MASK)
+#define LINE_SIZE_MASK(dcache)  ((dcache->line_size - 1))
+#define XFORM(dcache, x) 	((x) & LINE_SIZE_MASK(dcache))
+#define MASK(dcache, x)         ((x) & ~LINE_SIZE_MASK(dcache))
 
 struct dcache_block
 {
@@ -93,8 +94,8 @@ struct dcache_block
   struct dcache_block *next;
 
   CORE_ADDR addr;		/* address of data */
-  gdb_byte data[LINE_SIZE];	/* bytes at given address */
   int refs;			/* # hits */
+  gdb_byte data[0];		/* line_size bytes at given address */
 };
 
 struct dcache_struct
@@ -108,6 +109,7 @@ struct dcache_struct
 
   /* The number of in-use lines in the cache.  */
   int size;
+  CORE_ADDR line_size;  /* current line_size.  */
 
   /* The ptid of last inferior to use cache or null_ptid.  */
   ptid_t ptid;
@@ -207,6 +209,29 @@ for_each_block (struct dcache_block **bl
   while (*blist && db != *blist);
 }
 
+/* BLOCK_FUNC routine for dcache_free.  */
+
+static void
+free_block (struct dcache_block *block, void *param)
+{
+  free (block);
+}
+
+/* Free a data cache.  */
+
+void
+dcache_free (DCACHE *dcache)
+{
+  if (last_cache == dcache)
+    last_cache = NULL;
+
+  splay_tree_delete (dcache->tree);
+  for_each_block (&dcache->oldest, free_block, NULL);
+  for_each_block (&dcache->freelist, free_block, NULL);
+  xfree (dcache);
+}
+
+
 /* BLOCK_FUNC function for dcache_invalidate.
    This doesn't remove the block from the oldest list on purpose.
    dcache_invalidate will do it later.  */
@@ -230,6 +255,16 @@ dcache_invalidate (DCACHE *dcache)
   dcache->oldest = NULL;
   dcache->size = 0;
   dcache->ptid = null_ptid;
+
+  if (dcache->line_size != dcache_line_size)
+    {
+      /* We've been asked to use a different line size.
+	 All of our freelist blocks are now the wrong size, so free them.  */
+
+      for_each_block (&dcache->freelist, free_block, dcache);
+      dcache->freelist = NULL;
+      dcache->line_size = dcache_line_size;
+    }
 }
 
 /* Invalidate the line associated with ADDR.  */
@@ -257,7 +292,7 @@ dcache_hit (DCACHE *dcache, CORE_ADDR ad
   struct dcache_block *db;
 
   splay_tree_node node = splay_tree_lookup (dcache->tree,
-					    (splay_tree_key) MASK (addr));
+					    (splay_tree_key) MASK (dcache, addr));
 
   if (!node)
     return NULL;
@@ -281,7 +316,7 @@ dcache_read_line (DCACHE *dcache, struct
   int reg_len;
   struct mem_region *region;
 
-  len = LINE_SIZE;
+  len = dcache->line_size;
   memaddr = db->addr;
   myaddr  = db->data;
 
@@ -325,7 +360,7 @@ dcache_alloc (DCACHE *dcache, CORE_ADDR 
 {
   struct dcache_block *db;
 
-  if (dcache->size >= DCACHE_SIZE)
+  if (dcache->size >= dcache_size)
     {
       /* Evict the least recently allocated line.  */
       db = dcache->oldest;
@@ -339,12 +374,12 @@ dcache_alloc (DCACHE *dcache, CORE_ADDR 
       if (db)
 	remove_block (&dcache->freelist, db);
       else
-	db = xmalloc (sizeof (struct dcache_block));
+	db = xmalloc (sizeof (struct dcache_block) + dcache->line_size);
 
       dcache->size++;
     }
 
-  db->addr = MASK (addr);
+  db->addr = MASK (dcache, addr);
   db->refs = 0;
 
   /* Put DB at the end of the list, it's the newest.  */
@@ -374,7 +409,7 @@ dcache_peek_byte (DCACHE *dcache, CORE_A
          return 0;
     }
 
-  *ptr = db->data[XFORM (addr)];
+  *ptr = db->data[XFORM (dcache, addr)];
   return 1;
 }
 
@@ -395,7 +430,7 @@ dcache_poke_byte (DCACHE *dcache, CORE_A
   struct dcache_block *db = dcache_hit (dcache, addr);
 
   if (db)
-    db->data[XFORM (addr)] = *ptr;
+    db->data[XFORM (dcache, addr)] = *ptr;
 
   return 1;
 }
@@ -427,33 +462,13 @@ dcache_init (void)
   dcache->oldest = NULL;
   dcache->freelist = NULL;
   dcache->size = 0;
+  dcache->line_size = dcache_line_size;
   dcache->ptid = null_ptid;
   last_cache = dcache;
 
   return dcache;
 }
 
-/* BLOCK_FUNC routine for dcache_free.  */
-
-static void
-free_block (struct dcache_block *block, void *param)
-{
-  free (block);
-}
-
-/* Free a data cache.  */
-
-void
-dcache_free (DCACHE *dcache)
-{
-  if (last_cache == dcache)
-    last_cache = NULL;
-
-  splay_tree_delete (dcache->tree);
-  for_each_block (&dcache->oldest, free_block, NULL);
-  for_each_block (&dcache->freelist, free_block, NULL);
-  xfree (dcache);
-}
 
 /* Read or write LEN bytes from inferior memory at MEMADDR, transferring
    to or from debugger address MYADDR.  Write to inferior if SHOULD_WRITE is
@@ -571,12 +586,12 @@ dcache_print_line (int index)
   printf_filtered (_("Line %d: address %s [%d hits]\n"),
 		   index, paddress (target_gdbarch, db->addr), db->refs);
 
-  for (j = 0; j < LINE_SIZE; j++)
+  for (j = 0; j < last_cache->line_size; j++)
     {
       printf_filtered ("%02x ", db->data[j]);
 
       /* Print a newline every 16 bytes (48 characters).  */
-      if ((j % 16 == 15) && (j != LINE_SIZE - 1))
+      if ((j % 16 == 15) && (j != last_cache->line_size - 1))
 	printf_filtered ("\n");
     }
   printf_filtered ("\n");
@@ -603,8 +618,10 @@ dcache_info (char *exp, int tty)
       return;
     }
 
-  printf_filtered (_("Dcache line width %d, maximum size %d\n"),
-		   LINE_SIZE, DCACHE_SIZE);
+  printf_filtered (_("Dcache %u lines of %u bytes each.\n"),
+		   dcache_size,
+		   last_cache ? (unsigned) last_cache->line_size
+		   : dcache_line_size);
 
   if (!last_cache || ptid_equal (last_cache->ptid, null_ptid))
     {
@@ -635,6 +652,35 @@ dcache_info (char *exp, int tty)
   printf_filtered (_("Cache state: %d active lines, %d hits\n"), i, refcount);
 }
 
+static void
+set_dcache_size (char *args, int from_tty,
+		 struct cmd_list_element *c)
+{
+  if (dcache_size <= 0)
+    {
+      unsigned d = dcache_size;
+      dcache_size = DCACHE_DEFAULT_SIZE;
+      error (_("Invalid dcache size: %u (must be positive)."), d);
+    }
+  if (last_cache)
+    dcache_invalidate (last_cache);
+}
+
+static void
+set_dcache_line_size (char *args, int from_tty,
+		      struct cmd_list_element *c)
+{
+  if (dcache_line_size < 2
+      || (dcache_line_size & (dcache_line_size - 1)) != 0)
+    {
+      unsigned d = dcache_line_size;
+      dcache_line_size = DCACHE_DEFAULT_LINE_SIZE;
+      error (_("Invalid dcache line size: %u (must be power of 2)."), d);
+    }
+  if (last_cache)
+    dcache_invalidate (last_cache);
+}
+
 void
 _initialize_dcache (void)
 {
@@ -656,4 +702,21 @@ Print information on the dcache performa
 With no arguments, this command prints the cache configuration and a\n\
 summary of each line in the cache.  Use \"info dcache <lineno> to dump\"\n\
 the contents of a given line."));
+
+  add_setshow_uinteger_cmd ("dcache-line-size", class_obscure,
+			    &dcache_line_size, _("\
+Set dcache line size (in bytes, must be power of 2)."), _("\
+Show dcache line size."),
+			    NULL,
+			    set_dcache_line_size,
+			    NULL,
+			    &setlist, &showlist);
+  add_setshow_uinteger_cmd ("dcache-size", class_obscure,
+			    &dcache_size, _("\
+Set number of dcache lines."), _("\
+Show number of dcache lines."),
+			    NULL,
+			    set_dcache_size,
+			    NULL,
+			    &setlist, &showlist);
 }