[PATCH v2] Add dll trampoline code handling for windows 64bit

[Roland Schwingel <roland@onevision.com>]

[-- Attachment #1: Type: text/plain, Size: 1623 bytes --]

Hi...

Here is a new version of my patch from yesterday. I took (hopefully 
correct) all
suggestions regarding formatting and indention into account. If there is 
still
something wrong just tell me.

Regarding my byteswap code:
I spent nearly the whole day investigating that up and down. I can't use
the gdb read_memory functions here that do endianness conversion. I stepped
down many times the read_memory functions over all frames until the
point where the memory is read from the inferior using windows native
function ReadProcessMemory(). It appears that - in this case - the memory
returned is rotated by 2 bytes.

Example: Function pointer in a dll is eg: 0x0000000015027bba
In little endian it would be: 0xba7b021500000000
The ReadProcessMemory() call in windows-nat.c in function
windows_xfer_memory() always returns this:
0x021500000000ba7b
In all other cases the memory returned appears not to be rotated.
I also tried reading byte/word/int-wise but the result did not change.

I also queried the internet a long time but could not find any clear
explanation for this. So I left a comment in the patch at this point
as the result delivered by the new function represents the right
function pointer in any case I have checked.

Regarding Copyright Assignment:
As soon as I get the email from Tom I will do everything explained there 
and send it in.

ChangeLog:

2012-03-15  Roland Schwingel <roland.schwingel@onevision.com>

         * amd64-windows-tdep.c: #include "frame.h".
         (amd64_windows_skip_trampoline_code): New function.
         (amd64_windows_init_abi): Add trampoline registration.

Roland

[-- Attachment #2: amd64-windows-tdep.c.patch --]
[-- Type: text/plain, Size: 2665 bytes --]

--- amd64-windows-tdep.c_orig	2012-03-02 01:06:12.000000000 +0100
+++ amd64-windows-tdep.c	2012-03-15 15:37:45.647047400 +0100
@@ -23,6 +23,7 @@
 #include "gdbtypes.h"
 #include "gdbcore.h"
 #include "regcache.h"
+#include "frame.h"
 
 /* The registers used to pass integer arguments during a function call.  */
 static int amd64_windows_dummy_call_integer_regs[] =
@@ -153,12 +154,65 @@
   return pc;
 }
 
+/* Check win64 DLL jmp trampolines and find jump destination.  */
+
+static CORE_ADDR
+amd64_windows_skip_trampoline_code (struct frame_info *frame, CORE_ADDR pc)
+{
+  CORE_ADDR destination = 0;
+  struct gdbarch *gdbarch = get_frame_arch (frame);
+  enum bfd_endian byte_order = gdbarch_byte_order (gdbarch);
+
+  /* Check for jmp *<offset>(%rip) (jump near, absolute indirect (/4)).  */
+  if (pc && read_memory_unsigned_integer (pc, 2, byte_order) == 0x25ff)
+    {
+      /* Get opcode offset and see if we can find a reference in our data.  */
+      ULONGEST indirect =
+        read_memory_unsigned_integer (pc + 2, 4, byte_order);
+      struct minimal_symbol *indsym =
+        indirect ? lookup_minimal_symbol_by_pc (pc + indirect) : 0;
+      const char *symname = indsym ? SYMBOL_LINKAGE_NAME (indsym) : 0;
+
+      if (symname)
+        {
+          if (strncmp (symname, "__imp_", 6) == 0
+              || strncmp (symname, "_imp_", 5) == 0)
+            {	   	     
+              gdb_byte *pos,addr[8];
+
+              read_memory(pc + indirect, addr, 8);
+              /* The data fetched from the inferior is in this
+                 case not little endian, 2 bytes from the
+                 beginning are rotated to the end. 
+                 Example: 
+                 function pointer expected in little endian:
+                 0xba7b021500000000 
+                 pointer fetched from inferior:
+                 0x021500000000ba7b  
+                 So I do byteswapping here on my own. */
+              pos = (gdb_byte *)&destination;
+              pos[0] = addr[6];
+              pos[1] = addr[7];
+              pos[2] = addr[0];
+              pos[3] = addr[1];
+              pos[4] = addr[2];
+              pos[5] = addr[3];
+              pos[6] = addr[4];
+              pos[7] = addr[5];
+            }
+        }
+    }
+  return destination;
+}
 
 static void
 amd64_windows_init_abi (struct gdbarch_info info, struct gdbarch *gdbarch)
 {
   struct gdbarch_tdep *tdep = gdbarch_tdep (gdbarch);
 
+  /* register trampoline handling code */
+  set_gdbarch_skip_trampoline_code (gdbarch, amd64_windows_skip_trampoline_code);
+
   amd64_init_abi (info, gdbarch);
 
   /* On Windows, "long"s are only 32bit.  */