[RFA/commit 2/3] amd64-windows: memory args passed by pointer during function calls.

[Joel Brobecker <brobecker@adacore.com>]

From: brobecke <brobecke@f8352e7e-cb20-0410-8ce7-b5d9e71c585c>

Another difference between Linux and Windows on amd64: When an argument
is passed through the stack (aka MEMORY), the argument address must also
be passed in the associated integer register (if still available).

http://msdn.microsoft.com/en-us/library/zthk2dkh%28VS.80%29.aspx:

    __m128 types, arrays and strings are never passed by immediate value
    but rather a pointer will be passed to memory allocated by the caller.
    Structs/unions of size 8, 16, 32, or 64 bits and __m64 will
    be passed as if they were integers of the same size. Structs/unions
    other than these sizes will be passed as a pointer to memory
    allocated by the caller.

I should also mention that this affect return values.  This is actually
the situation that I started looking at, because I had a testcase where
a function taking 3 integers was called, and I thought that was the
simplest of all the function-call failures I was looking at:

    struct large
    {
      int x;
      int y;
      int z;
    };

    struct large
    create (int x, int y, int z)
    {
      struct large result = {x, y, z};

      return result;
    }

When I call create with 3 parameters, I get a SIGSEGV:

    (gdb) print create (74, 8, 4)

    Program received signal SIGSEGV, Segmentation fault.
    0x0000000000401695 in create (x=8, y=4, z=2293376) at bar.c:18
    18        return result;

(notice how the parameters get shifted to the left, since the function
thinks that the address of the return value is 74).

This patch enhances the amd64_push_arguments routine, allowing it to
follow either calling convention.  Which calling convention is used
depends on the value of a new field in struct gdbarch_tdep
(memory_args_by_pointer).  The default value is zero (Linux convention),
while amd64-windows-tdep sets this field to 1.

gdb/ChangeLog:

    * i386-tdep.h (gdbarch_tdep): Add field memory_args_by_pointer.
    * amd64-tdep.c (amd64_push_arguments): Add handling of architectures
    where tdep->memory_args_by_pointer is non-zero.
    * amd64-windows-tdep.c (amd64_windows_init_abi): Set
    tdep->memory_args_by_pointer to 1.

I can certainly submit a new testcase, but it looks like struct.exp
should cover much more than this case...

-- 
Joel

---
 gdb/amd64-tdep.c         |   37 +++++++++++++++++++++++++++++++++----
 gdb/amd64-windows-tdep.c |    1 +
 gdb/i386-tdep.h          |    9 +++++++++
 3 files changed, 43 insertions(+), 4 deletions(-)

diff --git a/gdb/amd64-tdep.c b/gdb/amd64-tdep.c
index f69f3f6..e9e0809 100644
--- a/gdb/amd64-tdep.c
+++ b/gdb/amd64-tdep.c
@@ -570,7 +570,8 @@ static CORE_ADDR
 amd64_push_arguments (struct regcache *regcache, int nargs,
 		      struct value **args, CORE_ADDR sp, int struct_return)
 {
-  struct gdbarch_tdep *tdep = gdbarch_tdep (get_regcache_arch (regcache));
+  struct gdbarch *gdbarch = get_regcache_arch (regcache);
+  struct gdbarch_tdep *tdep = gdbarch_tdep (gdbarch);
   int *integer_regs = tdep->call_dummy_integer_regs;
   int num_integer_regs = tdep->call_dummy_num_integer_regs;
 
@@ -583,6 +584,11 @@ amd64_push_arguments (struct regcache *regcache, int nargs,
     AMD64_XMM0_REGNUM + 6, AMD64_XMM0_REGNUM + 7,
   };
   struct value **stack_args = alloca (nargs * sizeof (struct value *));
+  /* An array that mirrors the stack_args array.  For all arguments
+     that are passed by MEMORY, if that argument's address also needs
+     to be stored in a register, the ARG_ADDR_REGNO array will contain
+     that register number (or a negative value otherwise).  */
+  int *arg_addr_regno = alloca (nargs * sizeof (int));
   int num_stack_args = 0;
   int num_elements = 0;
   int element = 0;
@@ -626,7 +632,19 @@ amd64_push_arguments (struct regcache *regcache, int nargs,
 	{
 	  /* The argument will be passed on the stack.  */
 	  num_elements += ((len + 7) / 8);
-	  stack_args[num_stack_args++] = args[i];
+	  stack_args[num_stack_args] = args[i];
+          /* If this is an AMD64_MEMORY argument whose address must also
+             be passed in one of the integer registers, reserve that
+             register and associate this value to that register so that
+             we can store the argument address as soon as we know it.  */
+          if (class[0] == AMD64_MEMORY
+              && tdep->memory_args_by_pointer
+              && integer_reg < tdep->call_dummy_num_integer_regs)
+            arg_addr_regno[num_stack_args] =
+              tdep->call_dummy_integer_regs[integer_reg++];
+          else
+            arg_addr_regno[num_stack_args] = -1;
+          num_stack_args++;
 	}
       else
 	{
@@ -682,8 +700,19 @@ amd64_push_arguments (struct regcache *regcache, int nargs,
       struct type *type = value_type (stack_args[i]);
       const gdb_byte *valbuf = value_contents (stack_args[i]);
       int len = TYPE_LENGTH (type);
-
-      write_memory (sp + element * 8, valbuf, len);
+      CORE_ADDR arg_addr = sp + element * 8;
+
+      write_memory (arg_addr, valbuf, len);
+      if (arg_addr_regno[i] >= 0)
+        {
+          /* We also need to store the address of that argument in
+             the given register.  */
+          gdb_byte buf[8];
+          enum bfd_endian byte_order = gdbarch_byte_order (gdbarch);
+
+          store_unsigned_integer (buf, 8, byte_order, arg_addr);
+          regcache_cooked_write (regcache, arg_addr_regno[i], buf);
+        }
       element += ((len + 7) / 8);
     }
 
diff --git a/gdb/amd64-windows-tdep.c b/gdb/amd64-windows-tdep.c
index b5a0035..0ed9368 100644
--- a/gdb/amd64-windows-tdep.c
+++ b/gdb/amd64-windows-tdep.c
@@ -83,6 +83,7 @@ amd64_windows_init_abi (struct gdbarch_info info, struct gdbarch *gdbarch)
     ARRAY_SIZE (amd64_windows_dummy_call_integer_regs);
   tdep->call_dummy_integer_regs = amd64_windows_dummy_call_integer_regs;
   tdep->classify = amd64_windows_classify;
+  tdep->memory_args_by_pointer = 1;
 
   set_solib_ops (gdbarch, &solib_target_so_ops);
 }
diff --git a/gdb/i386-tdep.h b/gdb/i386-tdep.h
index d5b24fa..f79a15d 100644
--- a/gdb/i386-tdep.h
+++ b/gdb/i386-tdep.h
@@ -86,6 +86,15 @@ struct gdbarch_tdep
      the result in CLASS.  Used on amd64 only.  */
   void (*classify) (struct type *type, enum amd64_reg_class class[2]);
 
+  /* Non-zero if the first few MEMORY arguments should be passed by
+     pointer.
+
+     More precisely, MEMORY arguments are passed through the stack.
+     But certain architectures require that their address be passed
+     by register as well, if there are still some integer registers
+     available for argument passing.  */
+  int memory_args_by_pointer;
+
   /* Floating-point registers.  */
   struct regset *fpregset;
   size_t sizeof_fpregset;
-- 
1.6.3.3