[PATCH 1/3] gdb, amd64: extend the amd64 prologue analyzer to skip register pushes

[Pawel Kupczak <pawel.kupczak@intel.com>]

A typical function's prologue can consist of setting up a frame pointer,
pushing registers onto the stack and allocating space on the stack.
Current amd64 prologue analyzer would stop after the frame setup.
This patch allows GDB to skip past register pushes, while also improving
unwinding pushed registers, for functions with a frame pointer, without
debug info and .cfi directives found in .eh_frame section that are used
for unwinding.  Skipping register pushes was also present for i386
targets before - the proposed changes are based on i386 implementation.

It also improves the unwinding even if .cfi directives are present,
because GDB can only unwind a register if it has reached a corresponding
.cfi directive, which won't be there before the pushes.

Additionally, at least gcc 11.4 and later by default doesn't emit
necessary debug info, which GDB would try to use to find prologue's end.
In that case, extended prologue analyzer would take effect.

Using C source listed below as an example, compiled with gcc 11.4.0:
```
	int __attribute__ ((noinline))
	bar (int a)
	{
	    return a + a;
	}

	int __attribute__ ((noinline))
	foo (int a, int b, int c, int d, int e)
	{
	    int x = bar (a) + bar (b) + bar (c) + bar (d) + bar (e);
	    return x;
	}

	int
	main (int argc, char **argv)
	{
	    return foo (1, 2, 3, 4, 5);
	}
```

Compiling with "gcc -O1 -fno-omit-frame-pointer
-fno-asynchronous-unwind-tables", we get:
```
	(gdb) b foo
	Breakpoint 1 at 0x1139
	(gdb) r
	...
	Breakpoint 1, 0x0000555555555139 in foo ()
	(gdb) disassemble
	Dump of assembler code for function foo:
	   0x0000555555555131 <+0>:     endbr64
	   0x0000555555555135 <+4>:     push   %rbp
	   0x0000555555555136 <+5>:     mov    %rsp,%rbp
	=> 0x0000555555555139 <+8>:     push   %r15
	   0x000055555555513b <+10>:    push   %r14
	   0x000055555555513d <+12>:    push   %r13
	   0x000055555555513f <+14>:    push   %r12
	   0x0000555555555141 <+16>:    push   %rbx
	   0x0000555555555142 <+17>:    sub    $0x8,%rsp
	   0x0000555555555146 <+21>:    mov    %esi,%r15d
	...
	(gdb) ni
	0x000055555555513b in foo ()
	(gdb) p $r15
	$1 = 140737354125376
	(gdb) p $r15=1234
	$2 = 1234
	(gdb) p $r15
	$3 = 1234
	(gdb) up
	#1  0x00005555555551b7 in main ()
	(gdb) p $r15
	$4 = 1234
```

With the proposed changes, breakpoint gets past those register pushes:
```
	(gdb) b foo
	Breakpoint 1 at 0x1142
	(gdb) r
	...
	Breakpoint 1, 0x0000555555555142 in foo ()
	(gdb) disassemble
	Dump of assembler code for function foo:
	   0x0000555555555131 <+0>:     endbr64
	   0x0000555555555135 <+4>:     push   %rbp
	   0x0000555555555136 <+5>:     mov    %rsp,%rbp
	   0x0000555555555139 <+8>:     push   %r15
	   0x000055555555513b <+10>:    push   %r14
	   0x000055555555513d <+12>:    push   %r13
	   0x000055555555513f <+14>:    push   %r12
	   0x0000555555555141 <+16>:    push   %rbx
	=> 0x0000555555555142 <+17>:    sub    $0x8,%rsp
	   0x0000555555555146 <+21>:    mov    %esi,%r15d
	...
```

Also, unwinding pushed registers now works:
```
	...
	Breakpoint 1, 0x0000555555555142 in foo ()
	(gdb) disassemble
	Dump of assembler code for function foo:
	   0x0000555555555131 <+0>:     endbr64
	   0x0000555555555135 <+4>:     push   %rbp
	   0x0000555555555136 <+5>:     mov    %rsp,%rbp
	   0x0000555555555139 <+8>:     push   %r15
	   0x0000555555555139 <+8>:     push   %r15
	   0x000055555555513b <+10>:    push   %r14
	   0x000055555555513d <+12>:    push   %r13
	   0x000055555555513f <+14>:    push   %r12
	   0x0000555555555141 <+16>:    push   %rbx
	=> 0x0000555555555142 <+17>:    sub    $0x8,%rsp
	   0x0000555555555146 <+21>:    mov    %esi,%r15d
	...
	(gdb) p $r15
	$1 = 140737354125376
	(gdb) p $r15=1234
	$2 = 1234
	(gdb) p $r15
	$3 = 1234
	(gdb) up
	#1  0x00005555555551b7 in main ()
	(gdb) p $r15
	$4 = 140737354125376
```

Additionally a new test was added to verify this behavior.
---
 gdb/amd64-tdep.c                              | 53 +++++++++++-
 .../amd64-extended-prologue-analysis.c        | 49 +++++++++++
 .../amd64-extended-prologue-analysis.exp      | 86 +++++++++++++++++++
 3 files changed, 187 insertions(+), 1 deletion(-)
 mode change 100644 => 100755 gdb/amd64-tdep.c
 create mode 100644 gdb/testsuite/gdb.arch/amd64-extended-prologue-analysis.c
 create mode 100644 gdb/testsuite/gdb.arch/amd64-extended-prologue-analysis.exp

diff --git a/gdb/amd64-tdep.c b/gdb/amd64-tdep.c
old mode 100644
new mode 100755
index 82dd1e07cf3..863b29a8c27
--- a/gdb/amd64-tdep.c
+++ b/gdb/amd64-tdep.c
@@ -2553,6 +2553,56 @@ amd64_analyze_frame_setup (gdbarch *gdbarch, CORE_ADDR pc,
   return pc;
 }
 
+/* Check whether PC points at code pushing registers onto the stack.  If so,
+   update CACHE and return pc after those pushes or CURRENT_PC, whichever is
+   smaller.  Otherwise, return PC passed to this function.  */
+
+static CORE_ADDR
+amd64_analyze_register_saves (CORE_ADDR pc, CORE_ADDR current_pc,
+			      amd64_frame_cache *cache)
+{
+  gdb_byte op;
+  int offset = 0;
+
+  /* There are at most 16 registers that would be pushed in the prologue.  */
+  for (int i = 0; i < 16 && pc < current_pc; i++)
+    {
+      int reg = 0;
+      int pc_offset = 0;
+
+      if (target_read_code (pc, &op, 1) == -1)
+	return pc;
+
+      /* %r8 - %r15 prefix.  */
+      if (op == 0x41)
+	{
+	  reg += 8;
+	  pc_offset = 1;
+
+	  if (target_read_code (pc + 1, &op, 1) == -1)
+	    return pc;
+	}
+
+      /* push %rax|%rcx|%rdx|%rbx|%rsp|%rbp|%rsi|%rdi
+
+	 or with 0x41 prefix:
+	 push %r8|%r9|%r10|%r11|%r12|%r13|%r14|%r15.  */
+      if (op < 0x50 || op > 0x57)
+	break;
+
+      reg += op - 0x50;
+      offset -= 8;
+
+      int regnum = amd64_arch_reg_to_regnum (reg);
+      cache->saved_regs[regnum] = offset;
+      cache->sp_offset += 8;
+
+      pc += 1 + pc_offset;
+    }
+
+  return pc;
+}
+
 /* Do a limited analysis of the prologue at PC and update CACHE
    accordingly.  Bail out early if CURRENT_PC is reached.  Return the
    address where the analysis stopped.
@@ -2594,7 +2644,8 @@ amd64_analyze_prologue (gdbarch *gdbarch, CORE_ADDR pc, CORE_ADDR current_pc,
   if (current_pc <= pc)
     return current_pc;
 
-  return amd64_analyze_frame_setup (gdbarch, pc, current_pc, cache);
+  pc = amd64_analyze_frame_setup (gdbarch, pc, current_pc, cache);
+  return amd64_analyze_register_saves (pc, current_pc, cache);
 }
 
 /* Work around false termination of prologue - GCC PR debug/48827.
diff --git a/gdb/testsuite/gdb.arch/amd64-extended-prologue-analysis.c b/gdb/testsuite/gdb.arch/amd64-extended-prologue-analysis.c
new file mode 100644
index 00000000000..7d777d23236
--- /dev/null
+++ b/gdb/testsuite/gdb.arch/amd64-extended-prologue-analysis.c
@@ -0,0 +1,49 @@
+/* This testcase is part of GDB, the GNU debugger.
+
+   Copyright 2025 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+int __attribute__ ((noinline))
+bar (int x)
+{
+  return x + x;
+}
+
+/* This function should generate a prologue in shape of:
+    push  %rbp
+    .cfi_def_cfa_offset 16
+    .cfi_offset %rbp, -16
+    mov   %rsp, %rbp
+    .cfi_def_cfa_register %rbp
+    push  %reg1
+    push  %reg2
+    .cfi_offset %reg2, 32
+    .cfi_offset %reg1, 24
+
+    So to be able to unwind a register, GDB needs to skip prologue past
+    register pushes (to access .cfi directives).  */
+int __attribute__ ((noinline))
+foo (int a, int b, int c, int d)
+{
+  a += bar (a) + bar (b) + bar (c);
+  return a;
+}
+
+int
+main (int argc, char **argv)
+{
+  int a = foo (argc, argc + 1, argc + 2, argc * 2);
+  return a;
+}
diff --git a/gdb/testsuite/gdb.arch/amd64-extended-prologue-analysis.exp b/gdb/testsuite/gdb.arch/amd64-extended-prologue-analysis.exp
new file mode 100644
index 00000000000..49e03357995
--- /dev/null
+++ b/gdb/testsuite/gdb.arch/amd64-extended-prologue-analysis.exp
@@ -0,0 +1,86 @@
+# Copyright 2025 Free Software Foundation, Inc.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This test verifies that when placing a breakpoint on a function with a frame
+# pointer, instructions that push callee-saved registers in the prologue are
+# skipped, without debug info.  When stopped on such breakpoint, the pushed
+# registers should be able to be immediately unwound.  With debug info present,
+# GDB would try to use prologue-end markers found in the line table to
+# determine where the prologue ends.
+#
+# It is also tested both with and without .eh_frame's .cfi directives - with
+# them, GDB can only unwind a register once stopped after .cfi directive for
+# that register's push.
+
+require is_x86_64_m64_target
+standard_testfile .c
+
+# Place a breakpoint at foo, go to it and test if %r12 can be unwound.
+proc test_run {} {
+    gdb_breakpoint "foo"
+    gdb_continue_to_breakpoint "Continue to foo"
+
+    gdb_test "backtrace" "#0  $::hex in foo\[^s\]+#1  $::hex in main\[^s\]+" \
+	"Verify backtrace output in foo"
+
+    set main_r12 [get_integer_valueof "\$r12" "null"]
+    gdb_assert { $main_r12 ne "null" }
+
+    set foo_r12 [expr "$main_r12 + 2"]
+    gdb_test "print \$r12=$foo_r12" "$foo_r12" "Set foo's %r12=$foo_r12"
+    gdb_test "up" ".*main.*" "Go up a frame from foo"
+    gdb_test "print \$r12" "$main_r12" "foo's %r12 unwound"
+    gdb_test "down" ".*foo.*" "Go back down a frame to foo"
+    gdb_test "print \$r12" "$foo_r12" "foo's %r12 unwound back"
+}
+
+with_test_prefix "w/o .cfi directives" {
+    # -fno-asynchronous-unwind-tables is needed to get rid of .cfi directives in
+    # .eh_frame section.
+    if { [gdb_can_simple_compile fno-asynchronous-unwind-tables \
+	{ void foo () { } } object -fno-asynchronous-unwind-tables] != 0 } {
+
+	# For at least gcc 11.4/clang 14.0.0 and later, -O1 makes them more
+	# eager to use registers in the prologue.
+	#
+	# At least gcc 11.4 and later by default does not generate full debug
+	# info, "nodebug" is there for other compilers.
+	if { [prepare_for_testing "failed to prepare" $testfile $srcfile \
+	    { optimize=-O1 nodebug
+	    additional_flags=-fno-asynchronous-unwind-tables
+	    additional_flags=-fno-omit-frame-pointer}] } {
+	    return -1
+	}
+
+	if { ![runto_main] } {
+	    return -1
+	}
+
+	test_run
+    }
+}
+
+with_test_prefix "with .cfi directives" {
+    if { [prepare_for_testing "failed to prepare" $testfile $srcfile \
+	{ optimize=-O1 nodebug additional_flags=-fno-omit-frame-pointer}] } {
+	return -1
+    }
+
+    if { ![runto_main] } {
+	return -1
+    }
+
+    test_run
+}
-- 
2.34.1

---------------------------------------------------------------------
Intel Technology Poland sp. z o.o.
ul. Slowackiego 173 | 80-298 Gdansk | Sad Rejonowy Gdansk Polnoc | VII Wydzial Gospodarczy Krajowego Rejestru Sadowego - KRS 101882 | NIP 957-07-52-316 | Kapital zakladowy 200.000 PLN.
Spolka oswiadcza, ze posiada status duzego przedsiebiorcy w rozumieniu ustawy z dnia 8 marca 2013 r. o przeciwdzialaniu nadmiernym opoznieniom w transakcjach handlowych.

Ta wiadomosc wraz z zalacznikami jest przeznaczona dla okreslonego adresata i moze zawierac informacje poufne. W razie przypadkowego otrzymania tej wiadomosci, prosimy o powiadomienie nadawcy oraz trwale jej usuniecie; jakiekolwiek przegladanie lub rozpowszechnianie jest zabronione.
This e-mail and any attachments may contain confidential material for the sole use of the intended recipient(s). If you are not the intended recipient, please contact the sender and delete all copies; any review or distribution by others is strictly prohibited.