[PATCH] Add dll trampoline code handling for windows 64bit

[Roland Schwingel <roland@onevision.com>]

[-- Attachment #1: Type: text/plain, Size: 1251 bytes --]

Hi...

When single stepping a 64bit windows application gdb at present does not 
step into
functions residing in a dll. This is due to the fact that handling of 
dll trampoline code
for win64 is missing.

I added a new function to amd64-windows-tdep.c to handle this similar to 
the existing function (i386_pe_skip_trampoline_code()). With some 
differences:

- On 32bit windows dll trampoline code is expressed as jmp *(dest) while on
   64bit windows this is expressed as jmp *<offset>(%rip). Took care of 
this.
- The jump destination is on 64bit windows of course 8 byte long. I 
could not
   find a function that transforms this into a CORE_ADDR like 
read_memory_unsigned_integer()
   it is doing in the 32bit case.  So I did the transformation on my 
own. While this is
   high performant it might not be the "official" gdb way. If someone 
can give me a
   hint on how to the transformation the "official" way I will adjust my 
patch - if wished.

Now single stepping into dll code works.

ChangeLog:

2012-03-14  Roland Schwingel<roland.schwingel@onevision.com>

         * amd64-windows-tdep.c: #include "frame.h"
         (amd64_windows_skip_trampoline_code): New function.
	(amd64_windows_init_abi): Add trampoline registration.

Roland



[-- Attachment #2: amd64-windows-tdep.c.patch --]
[-- Type: text/plain, Size: 2086 bytes --]

--- amd64-windows-tdep.c_orig	2012-03-02 01:06:12.000000000 +0100
+++ amd64-windows-tdep.c	2012-03-14 13:31:39.815727600 +0100
@@ -23,6 +23,7 @@
 #include "gdbtypes.h"
 #include "gdbcore.h"
 #include "regcache.h"
+#include "frame.h"
 
 /* The registers used to pass integer arguments during a function call.  */
 static int amd64_windows_dummy_call_integer_regs[] =
@@ -153,12 +154,59 @@
   return pc;
 }
 
+/* Stuff for WIN64 PE style DLL's but is pretty generic really.  */
+
+static CORE_ADDR
+amd64_windows_skip_trampoline_code (struct frame_info *frame, CORE_ADDR pc)
+{
+    struct gdbarch     *gdbarch = get_frame_arch (frame);
+    enum bfd_endian     byte_order = gdbarch_byte_order (gdbarch);
+
+   /* check for jmp *<offset>(%rip) */
+    if (pc && read_memory_unsigned_integer (pc, 2, byte_order) == 0x25ff)
+      {
+	  unsigned long indirect =
+	    read_memory_unsigned_integer (pc + 2, 4, byte_order);
+	  struct minimal_symbol *indsym =
+	    indirect ? lookup_minimal_symbol_by_pc (pc + indirect) : 0;
+	  const char *symname =
+	    indsym ? SYMBOL_LINKAGE_NAME (indsym) : 0;
+
+	  if (symname)
+	    {
+		if (strncmp (symname, "__imp_", 6) == 0
+		    || strncmp (symname, "_imp_", 5) == 0)
+		  {
+		      CORE_ADDR           destination;
+		      gdb_byte           *pos, addr[8];
+
+		      read_memory (pc + indirect, addr, 8);
+		      pos = (gdb_byte *) &destination;
+		      pos[0] = addr[6];
+		      pos[1] = addr[7];
+		      pos[2] = addr[0];
+		      pos[3] = addr[1];
+		      pos[4] = addr[2];
+		      pos[5] = addr[3];
+		      pos[6] = addr[4];
+		      pos[7] = addr[5];
+
+		      return destination;
+		  }
+	    }
+      }
+    return 0;			/* Not a trampoline.  */
+}
 
 static void
 amd64_windows_init_abi (struct gdbarch_info info, struct gdbarch *gdbarch)
 {
   struct gdbarch_tdep *tdep = gdbarch_tdep (gdbarch);
 
+   /* register trampoline handling code */
+    set_gdbarch_skip_trampoline_code (gdbarch,
+				      amd64_windows_skip_trampoline_code);
+
   amd64_init_abi (info, gdbarch);
 
   /* On Windows, "long"s are only 32bit.  */