[PATCH 3/3] gdb, amd64: extend the amd64 prologue analyzer to skip stack alloc

[Pawel Kupczak <pawel.kupczak@intel.com>]

Following the previous patch (gdb, amd64: extend the amd64 prologue
analyzer to skip register pushes), this patch extends the analyzer
further to be able to skip stack space allocation as the next prologue
part, for functions with a frame pointer.  Implementation was based
on the i386 counterpart, which already had that functionality.

As of now, the stack allocation is not skipped.  Examples below use C
source listed below, compiled with gcc 11.4.0.
```
	int foo (int n)
	{
	    int ns[] = { 1, 4, 9, 16, 25 };
	    return ns[n];
	}

	int
	main (int argc, char **argv)
	{
	    return foo (argc);
	}
```

Compiling with "gcc -O0 -fno-omit-frame-pointer" we get:
```
	(gdb) b foo
	Breakpoint 1 at 0x1151
	(gdb) r
	...
	Breakpoint 1, 0x0000555555555151 in foo ()
	(gdb) disassemble
	Dump of assembler code for function foo:
	   0x0000555555555149 <+0>:     endbr64
	   0x000055555555514d <+4>:     push   %rbp
	   0x000055555555514e <+5>:     mov    %rsp,%rbp
	=> 0x0000555555555151 <+8>:     sub    $0x30,%rsp
	   0x0000555555555155 <+12>:    mov    %edi,-0x24(%rbp)
	...
```

With this patch, it gets skipped the same way register pushes are:
```
	(gdb) b foo
	Breakpoint 1 at 0x1155
	(gdb) r
	...
	Breakpoint 1, 0x0000555555555155 in foo ()
	(gdb) disassemble
	Dump of assembler code for function foo:
	   0x0000555555555149 <+0>:     endbr64
	   0x000055555555514d <+4>:     push   %rbp
	   0x000055555555514e <+5>:     mov    %rsp,%rbp
	   0x0000555555555151 <+8>:     sub    $0x30,%rsp
	=> 0x0000555555555155 <+12>:    mov    %edi,-0x24(%rbp)
	...
```
---
 gdb/amd64-tdep.c                              | 70 ++++++++++++++++++-
 .../amd64-extended-prologue-analysis.c        | 11 ++-
 .../amd64-extended-prologue-analysis.exp      | 10 +--
 3 files changed, 83 insertions(+), 8 deletions(-)

diff --git a/gdb/amd64-tdep.c b/gdb/amd64-tdep.c
index e4eb076a55a..a147d07d9b6 100755
--- a/gdb/amd64-tdep.c
+++ b/gdb/amd64-tdep.c
@@ -2603,6 +2603,70 @@ amd64_analyze_register_saves (CORE_ADDR pc, CORE_ADDR current_pc,
   return pc;
 }
 
+/* Check whether PC points at code allocating space on the stack.
+   If so, update CACHE and return pc past it or CURRENT_PC, whichever is
+   smaller.  Otherwise, return PC passed to this function.  */
+
+static CORE_ADDR
+amd64_analyze_stack_alloc (gdbarch *arch, CORE_ADDR pc, CORE_ADDR current_pc,
+			   amd64_frame_cache *cache)
+{
+  static const gdb_byte sub_imm8_rsp[]  = { 0x83, 0xec };
+  static const gdb_byte sub_imm32_rsp[] = { 0x81, 0xec };
+  static const gdb_byte lea_disp_rsp[]  = { 0x8D, 0x64 };
+
+  bfd_endian byte_order = gdbarch_byte_order (arch);
+  const CORE_ADDR start_pc = pc;
+
+  gdb_byte op;
+  if (target_read_code (pc, &op, 1) == -1)
+    return pc;
+
+  /* Check for REX.W.  */
+  if (op == 0x48)
+    pc++;
+
+  if (current_pc <= pc)
+    return current_pc;
+
+  gdb_byte buf[2];
+  read_code (pc, buf, 2);
+
+  /* Check for instruction allocating space on the stack, which looks like
+       sub imm8/32, %rsp
+     or
+       lea -imm (%rsp), %rsp
+
+     and forward pc past it + update cache.  */
+
+  /* sub imm8, %rsp.  */
+  if (memcmp (buf, sub_imm8_rsp, 2) == 0)
+    {
+      /* Instruction is 3 bytes long.  The imm8 arg is the 3rd, single
+	 byte.  */
+      cache->sp_offset += read_code_integer (pc + 2, 1, byte_order);
+      return pc + 3;
+    }
+  /* sub imm32, %rsp.  */
+  else if (memcmp (buf, sub_imm32_rsp, 2) == 0)
+    {
+      /* Instruction is 6 bytes long.  The imm32 arg is stored in 4 bytes,
+	 starting from 3rd one.  */
+      cache->sp_offset += read_code_integer (pc + 2, 4, byte_order);
+      return pc + 6;
+    }
+  /* lea -imm (%rsp), %rsp.  */
+  else if (memcmp (buf, lea_disp_rsp, 2) == 0)
+    {
+      /* Instruction is 4 bytes long.  The imm arg is the 4th, single
+	 byte.  */
+      cache->sp_offset += -1 * read_code_integer (pc + 3, 1, byte_order);
+      return pc + 4;
+    }
+
+  return start_pc;
+}
+
 /* Do a limited analysis of the prologue at PC and update CACHE
    accordingly.  Bail out early if CURRENT_PC is reached.  Return the
    address where the analysis stopped.
@@ -2648,7 +2712,11 @@ amd64_analyze_prologue (gdbarch *gdbarch, CORE_ADDR pc, CORE_ADDR current_pc,
   if (current_pc <= pc)
     return current_pc;
 
-  return amd64_analyze_register_saves (pc, current_pc, cache);
+  pc = amd64_analyze_register_saves (pc, current_pc, cache);
+  if (current_pc <= pc)
+    return current_pc;
+
+  return amd64_analyze_stack_alloc (gdbarch, pc, current_pc, cache);
 }
 
 /* Work around false termination of prologue - GCC PR debug/48827.
diff --git a/gdb/testsuite/gdb.arch/amd64-extended-prologue-analysis.c b/gdb/testsuite/gdb.arch/amd64-extended-prologue-analysis.c
index 7d777d23236..e4805ec6f55 100644
--- a/gdb/testsuite/gdb.arch/amd64-extended-prologue-analysis.c
+++ b/gdb/testsuite/gdb.arch/amd64-extended-prologue-analysis.c
@@ -29,15 +29,22 @@ bar (int x)
     .cfi_def_cfa_register %rbp
     push  %reg1
     push  %reg2
+    sub   $XXX, %rsp
     .cfi_offset %reg2, 32
     .cfi_offset %reg1, 24
 
     So to be able to unwind a register, GDB needs to skip prologue past
-    register pushes (to access .cfi directives).  */
+    register pushes and stack allocation (to access .cfi directives).  */
 int __attribute__ ((noinline))
 foo (int a, int b, int c, int d)
 {
-  a += bar (a) + bar (b) + bar (c);
+  /* "volatile" alone isn't enough for clang to not optimize it out and
+     allocate space on the stack.  */
+  volatile char s[256];
+  for (int i = 0; i < 256; i++)
+    s[i] = (char) (a + i);
+
+  a += bar (a) + bar (b) + bar (c) + bar (d);
   return a;
 }
 
diff --git a/gdb/testsuite/gdb.arch/amd64-extended-prologue-analysis.exp b/gdb/testsuite/gdb.arch/amd64-extended-prologue-analysis.exp
index 49e03357995..5aeff028a6c 100644
--- a/gdb/testsuite/gdb.arch/amd64-extended-prologue-analysis.exp
+++ b/gdb/testsuite/gdb.arch/amd64-extended-prologue-analysis.exp
@@ -14,11 +14,11 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This test verifies that when placing a breakpoint on a function with a frame
-# pointer, instructions that push callee-saved registers in the prologue are
-# skipped, without debug info.  When stopped on such breakpoint, the pushed
-# registers should be able to be immediately unwound.  With debug info present,
-# GDB would try to use prologue-end markers found in the line table to
-# determine where the prologue ends.
+# pointer, instructions that push callee-saved registers and stack allocation
+# in the prologue are skipped, without debug info.  When stopped on such
+# breakpoint, the pushed registers should be able to be immediately unwound.
+# With debug info present, GDB would try to use prologue-end markers found in
+# the line table to determine where the prologue ends.
 #
 # It is also tested both with and without .eh_frame's .cfi directives - with
 # them, GDB can only unwind a register once stopped after .cfi directive for
-- 
2.34.1

---------------------------------------------------------------------
Intel Technology Poland sp. z o.o.
ul. Slowackiego 173 | 80-298 Gdansk | Sad Rejonowy Gdansk Polnoc | VII Wydzial Gospodarczy Krajowego Rejestru Sadowego - KRS 101882 | NIP 957-07-52-316 | Kapital zakladowy 200.000 PLN.
Spolka oswiadcza, ze posiada status duzego przedsiebiorcy w rozumieniu ustawy z dnia 8 marca 2013 r. o przeciwdzialaniu nadmiernym opoznieniom w transakcjach handlowych.

Ta wiadomosc wraz z zalacznikami jest przeznaczona dla okreslonego adresata i moze zawierac informacje poufne. W razie przypadkowego otrzymania tej wiadomosci, prosimy o powiadomienie nadawcy oraz trwale jej usuniecie; jakiekolwiek przegladanie lub rozpowszechnianie jest zabronione.
This e-mail and any attachments may contain confidential material for the sole use of the intended recipient(s). If you are not the intended recipient, please contact the sender and delete all copies; any review or distribution by others is strictly prohibited.