[RFA] Support for x86 on-stack trampolines

[Jerome Guitton <guitton@adacore.com>]

On x86 cross targets, the debugger has trouble stepping over the following
line of Ada code (initialization of a class-wide object):

    S: Shape'Class := R;

Here is what happens:
   (gdb) n
   0x60c49e8a in ?? ()

The program jumps to this code location, which is a trampoline
located on the stack (there is an implicit call to a routine internally
generated by the Ada expander). As it is on the stack, GDB is naturally
unable to find the bounds of the current function, or any debugging
information, and is thus unable to continue.

This patch adds support for this sort of trampoline. It re-uses some
code from prologue scan (insn pattern matching). Tested on x86-linux,
no new regression. OK to apply?

gdb/ChangeLog:

	* i386-tdep.c (i386_in_stack_tramp_p, i386_stack_tramp_frame_sniffer):
	New functions.
	(i386_stack_tramp_frame_unwind): New static global.
	(i386_match_pattern): New function, extracted from i386_match_insn.
	(i386_find_insn): Renaming of i386_match_insn. Use i386_match_pattern.
	(i386_match_insn_block): New function.
	(i386_analyze_frame_setup): Sync with renaming of i386_match_insn.
	(i386_tramp_chain_in_reg_insns)
	(i386_insn i386_tramp_chain_on_stack_insns): New static variables.
	(i386_gdbarch_init): Add i386_stack_tramp_frame_unwind to list
	of unwinders.
---
 gdb/i386-tdep.c |  183 +++++++++++++++++++++++++++++++++++++++++++++++--------
 1 files changed, 157 insertions(+), 26 deletions(-)

diff --git a/gdb/i386-tdep.c b/gdb/i386-tdep.c
index 5f4089b..13cbc5b 100644
--- a/gdb/i386-tdep.c
+++ b/gdb/i386-tdep.c
@@ -1126,47 +1126,96 @@ struct i386_insn
   gdb_byte mask[I386_MAX_MATCHED_INSN_LEN];
 };
 
-/* Search for the instruction at PC in the list SKIP_INSNS.  Return
-   the first instruction description that matches.  Otherwise, return
-   NULL.  */
+/* Return whether instruction at PC matches PATTERN.  */
 
-static struct i386_insn *
-i386_match_insn (CORE_ADDR pc, struct i386_insn *skip_insns)
+static int
+i386_match_pattern (CORE_ADDR pc, struct i386_insn pattern)
 {
-  struct i386_insn *insn;
   gdb_byte op;
 
   if (target_read_memory (pc, &op, 1))
-    return NULL;
+    return 0;
 
-  for (insn = skip_insns; insn->len > 0; insn++)
+  if ((op & pattern.mask[0]) == pattern.insn[0])
     {
-      if ((op & insn->mask[0]) == insn->insn[0])
-	{
-	  gdb_byte buf[I386_MAX_MATCHED_INSN_LEN - 1];
-	  int insn_matched = 1;
-	  size_t i;
+      gdb_byte buf[I386_MAX_MATCHED_INSN_LEN - 1];
+      int insn_matched = 1;
+      size_t i;
 
-	  gdb_assert (insn->len > 1);
-	  gdb_assert (insn->len <= I386_MAX_MATCHED_INSN_LEN);
+      gdb_assert (pattern.len > 1);
+      gdb_assert (pattern.len <= I386_MAX_MATCHED_INSN_LEN);
 
-	  if (target_read_memory (pc + 1, buf, insn->len - 1))
-	    return NULL;
-
-	  for (i = 1; i < insn->len; i++)
-	    {
-	      if ((buf[i - 1] & insn->mask[i]) != insn->insn[i])
-		insn_matched = 0;
-	    }
+      if (target_read_memory (pc + 1, buf, pattern.len - 1))
+	return 0;
 
-	  if (insn_matched)
-	    return insn;
+      for (i = 1; i < pattern.len; i++)
+	{
+	  if ((buf[i - 1] & pattern.mask[i]) != pattern.insn[i])
+	    insn_matched = 0;
 	}
+      return insn_matched;
     }
+  return 0;
+}
+
+/* Search for the instruction at PC in the list INSN_PATTERNS.  Return
+   the first instruction description that matches.  Otherwise, return
+   NULL.  If a non-null value is returned, IX points to the matching
+   entry.  */
 
+static struct i386_insn *
+i386_find_insn (CORE_ADDR pc, struct i386_insn *insn_patterns, int *ix)
+{
+  struct i386_insn *pattern;
+
+  for (pattern = insn_patterns, *ix = 0;
+       pattern->len > 0;
+       pattern++, *ix = *ix + 1)
+    {
+      if (i386_match_pattern (pc, *pattern))
+	return pattern;
+    }
+
+  ix = 0;
   return NULL;
 }
 
+/* Return whether PC points inside a sequence of instructions that
+   matches INSN_PATTERNS.  */
+
+static int
+i386_match_insn_block (CORE_ADDR pc, struct i386_insn *insn_patterns)
+{
+  CORE_ADDR current_pc;
+  int ix, i;
+  gdb_byte op;
+  struct i386_insn *insn;
+
+  insn = i386_find_insn (pc, insn_patterns, &ix);
+  if (insn == NULL)
+    return 0;
+
+  current_pc = pc - insn->len;
+  for (i = ix - 1; i >= 0; i--)
+    {
+      if (!i386_match_pattern (current_pc, insn_patterns[i]))
+	return 0;
+
+      current_pc -= insn_patterns[i].len;
+    }
+
+  current_pc = pc + insn->len;
+  for (insn = insn_patterns + ix + 1; insn->len > 0; insn++)
+    {
+      if (!i386_match_pattern (current_pc, *insn))
+	return 0;
+
+      current_pc += insn->len;
+    }
+
+  return 1;
+}
+
 /* Some special instructions that might be migrated by GCC into the
    part of the prologue that sets up the new stack frame.  Because the
    stack frame hasn't been setup yet, no registers have been saved
@@ -1287,6 +1336,7 @@ i386_analyze_frame_setup (struct gdbarch *gdbarch,
   struct i386_insn *insn;
   gdb_byte op;
   int skip = 0;
+  int ix;
 
   if (limit <= pc)
     return limit;
@@ -1316,7 +1366,7 @@ i386_analyze_frame_setup (struct gdbarch *gdbarch,
 	 `movl %esp, %ebp' that actually sets up the frame.  */
       while (pc + skip < limit)
 	{
-	  insn = i386_match_insn (pc + skip, i386_frame_setup_skip_insns);
+	  insn = i386_find_insn (pc + skip, i386_frame_setup_skip_insns, &ix);
 	  if (insn == NULL)
 	    break;
 
@@ -1938,6 +1988,86 @@ static const struct frame_unwind i386_epilogue_frame_unwind =
 };
 \f
 
+/* Stack-based trampolines.  */
+
+/* These trampolines are used on cross x86 targets, when taking the
+   address of a nested function.  When executing these trampolines,
+   no stack frame is set up, so we are in a similar situation as in
+   epilogues and i386_epilogue_frame_this_id can be re-used.
+*/
+
+/* Static chain passed in register */
+
+struct i386_insn i386_tramp_chain_in_reg_insns[] =
+{
+  /* mov   <imm>, %ecx (or %eax, in case of fastcall) */
+  {5, {0xb8}, {0xfe}},
+
+  /* jmp   <imm> */
+  {5, {0xe9}, {0xff}},
+
+  {0}
+};
+
+/* Static chain passed on stack (when regparm=3) */
+
+struct i386_insn i386_tramp_chain_on_stack_insns[] =
+{
+  /* push   <imm> */
+  {5, {0x68}, {0xff}},
+
+  /* jmp   <imm> */
+  {5, {0xe9}, {0xff}},
+
+  {0}
+};
+
+/* Return whether PC points inside a stack trampoline.   */
+
+static int
+i386_in_stack_tramp_p (struct gdbarch *gdbarch, CORE_ADDR pc)
+{
+  gdb_byte insn;
+  char *name;
+
+  if (target_read_memory (pc, &insn, 1))
+    return 0;
+
+  /* A stack trampoline is detected if no name is associated
+    to the current pc and if it points inside a trampoline
+    sequence.  */
+
+  if (!i386_match_insn_block (pc, i386_tramp_chain_in_reg_insns)
+      && !i386_match_insn_block (pc, i386_tramp_chain_on_stack_insns))
+    return 0;
+
+  find_pc_partial_function (pc, &name, NULL, NULL);
+  return !name;
+}
+
+static int
+i386_stack_tramp_frame_sniffer (const struct frame_unwind *self,
+			     struct frame_info *this_frame,
+			     void **this_prologue_cache)
+{
+  if (frame_relative_level (this_frame) == 0)
+    return i386_in_stack_tramp_p (get_frame_arch (this_frame),
+				  get_frame_pc (this_frame));
+  else
+    return 0;
+}
+
+static const struct frame_unwind i386_stack_tramp_frame_unwind =
+{
+  NORMAL_FRAME,
+  default_frame_unwind_stop_reason,
+  i386_epilogue_frame_this_id,
+  i386_frame_prev_register,
+  NULL, 
+  i386_stack_tramp_frame_sniffer
+};
+\f
+
 /* Signal trampolines.  */
 
 static struct i386_frame_cache *
@@ -7295,6 +7425,7 @@ i386_gdbarch_init (struct gdbarch_info info, struct gdbarch_list *arches)
     tdep->mm0_regnum = -1;
 
   /* Hook in the legacy prologue-based unwinders last (fallback).  */
+  frame_unwind_append_unwinder (gdbarch, &i386_stack_tramp_frame_unwind);
   frame_unwind_append_unwinder (gdbarch, &i386_sigtramp_frame_unwind);
   frame_unwind_append_unwinder (gdbarch, &i386_frame_unwind);
 
-- 
1.7.0.2