[PATCH V2 1/2] AMD64, Prologue: Recognize stack decrementation as prologue operation.

[Bernhard Heckel <bernhard.heckel@intel.com>]

GCC, ICC and Clang decrement stack pointer within the prologue
sequence in order to reserve memory for local variables.
Recognize this subtraction to stop at the very end of the
prologue.

2016-12-16  Bernhard Heckel  <bernhard.heckel@intel.com>

gdb/Changelog:
	* amd64-tdep.c (amd64_analyze_prologue): Recognize stack decrementation
	  as prologue operation.

---
 gdb/amd64-tdep.c | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/gdb/amd64-tdep.c b/gdb/amd64-tdep.c
index a3a1fde..cbcddcb 100644
--- a/gdb/amd64-tdep.c
+++ b/gdb/amd64-tdep.c
@@ -2261,11 +2261,19 @@ amd64_x32_analyze_stack_align (CORE_ADDR pc, CORE_ADDR current_pc,
 
       pushq %rbp        0x55
       movq %rsp, %rbp   0x48 0x89 0xe5 (or 0x48 0x8b 0xec)
+      in addition, functions containing local variables
+	sub imm8, %rsp   0x48 0x83 0xec
+      or
+	sub imm32, %rsp   0x48 0x81 0xec
 
    or (for the X32 ABI):
 
       pushq %rbp        0x55
       movl %esp, %ebp   0x89 0xe5 (or 0x8b 0xec)
+      in addition, functions containing local variables
+	sub imm8, %esp   0x83 0xec
+      or
+	sub imm32, %esp   0x81 0xec
 
    Any function that doesn't start with one of these sequences will be
    assumed to have no prologue and thus no valid frame pointer in
@@ -2283,6 +2291,12 @@ amd64_analyze_prologue (struct gdbarch *gdbarch,
   /* Ditto for movl %esp, %ebp.  */
   static const gdb_byte mov_esp_ebp_1[2] = { 0x89, 0xe5 };
   static const gdb_byte mov_esp_ebp_2[2] = { 0x8b, 0xec };
+  /* Ditto for subtraction on the stack pointer.  */
+  static const gdb_byte sub_rsp_imm8[3] = { 0x48, 0x83, 0xec };
+  static const gdb_byte sub_rsp_imm32[3] = { 0x48, 0x81, 0xec };
+  /* Ditto for subtraction on the stack pointer.  */
+  static const gdb_byte sub_esp_imm8[2] = { 0x83, 0xec };
+  static const gdb_byte sub_esp_imm32[2] = { 0x81, 0xec };
 
   gdb_byte buf[3];
   gdb_byte op;
@@ -2316,6 +2330,18 @@ amd64_analyze_prologue (struct gdbarch *gdbarch,
 	{
 	  /* OK, we actually have a frame.  */
 	  cache->frameless_p = 0;
+
+	  /* GCC, ICC and Clang do subtraction on the stack pointer
+	     to reserve memory for local variables.
+	     Two common variants exist to do so.  */
+	  read_code (pc + 4, buf, 3);
+	  if (memcmp (buf, sub_rsp_imm8, 3) == 0)
+	    /* Operand is 1 byte.  */
+	    return pc + 8;
+	  else if (memcmp (buf, sub_rsp_imm32, 3) == 0)
+	    /* Operand is 4 bytes.  */
+	    return pc + 11;
+
 	  return pc + 4;
 	}
 
@@ -2327,6 +2353,18 @@ amd64_analyze_prologue (struct gdbarch *gdbarch,
 	    {
 	      /* OK, we actually have a frame.  */
 	      cache->frameless_p = 0;
+
+	      /* GCC, ICC and Clang do subtraction on the stack pointer
+		 to reserve memory for local variables.
+		 Two common variants exist to do so.  */
+	      read_code (pc + 3, buf, 2);
+	      if (memcmp (buf, sub_esp_imm8, 2) == 0)
+		/* Operand is 1 byte.  */
+		return pc + 6;
+	      else if (memcmp (buf, sub_esp_imm32, 2) == 0)
+		/* Operand is 4 bytes.  */
+		return pc + 9;
+
 	      return pc + 3;
 	    }
 	}
-- 
2.7.1.339.g0233b80