[RFA] Alpha/Tru64 maximum debuggable stack frame size

[Jerome Guitton <guitton@adacore.com>]

Alpha/Tru64's mdebug format does not support frames that are greater than
512 Kbytes. In such a case, GCC explicitly emits a null stack size.

This case confuses GDB; it assumes that 0 means frameless. And then
cannot unwind properly, obviously.

The following Ada program reproduces the problem:

--

procedure Big_Stack_Frame is

  type Big_Record is
    record
      Data : STRING (1 .. 262144);
    end record;

  procedure P1
    (V1 : out Big_Record;
     V2 : out Big_Record;
     V3 : out Big_Record;
     V4 : out Big_Record;
     V5 : out Big_Record;
     V6 : out Big_Record;
     V  : Big_Record) is
  begin
     null;  -- BREAK
  end P1;

  procedure P2 is
     V1 : Big_Record;
     V2 : Big_Record;
     V3 : Big_Record;
     V4 : Big_Record;
     V5 : Big_Record;
     V6 : Big_Record;
     V  : Big_Record := (Data => (others => ' '));
  begin
    P1 (V1 => V1,
        V2 => V2,
        V3 => V3,
        V4 => V4,
        V5 => V5,
        V6 => V6,
        V  => V);
  end P2;

begin
   P2;
end Big_Stack_Frame;

--

After reaching P2, I get the following backtrace:

[...]
(gdb)
    at big_stack_frame.adb:17
warning: Hit beginning of text section without finding enclosing function for address 0x11ffbbfa0
[...]

This patch implements an heuristic to discriminate between the two
cases (frameless or max frame size); and it improves the prolog
analyzer to handle the case of huge stacks (in such a case, a probing
loop is generated by GCC).

We have some problems to get a full run of the DejaGNU testsuite at
AdaCore on our Alpha/Tru64 machines; but this patch has been tested
against our internal testsuite without a problem. I can provide a
testcase if needed, based on the program above. I probably
won't be able to run it on Tru64 though.

OK to apply?

gdb/ChangeLog:

2010-04-22  Jerome Guitton  <guitton@adacore.com>

	* alpha-tdep.c (INSN_OPCODE, MEM_RA, MEM_RB, MEM_DISP, BR_RA)
	(OPR_FUNCTION, OPR_HAS_IMMEDIATE, OPR_RA, OPR_RC, OPR_LIT): New macros.
	(lda_opcode, stq_opcode, bne_opcode, subq_opcode, subq_function):
	New constants.
	(alpha_heuristic_analyze_probing_loop): New function.
	(alpha_heuristic_frame_unwind_cache): In the prologue analysis, detect
	and handle cases when a stack probe loop is generated.
	* alpha-mdebug-tdep.c (alpha_mdebug_frameless): New function.
	(alpha_mdebug_max_frame_size_exceeded): New function.
	(alpha_mdebug_after_prologue): Use alpha_mdebug_frameless.
	(alpha_mdebug_frame_sniffer, alpha_mdebug_frame_base_sniffer):
        Return 0 when the maximum debuggable frame size has been exceeded.
---
 gdb/alpha-mdebug-tdep.c |   36 ++++++++++++-
 gdb/alpha-tdep.c        |  133 +++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 167 insertions(+), 2 deletions(-)

diff --git a/gdb/alpha-mdebug-tdep.c b/gdb/alpha-mdebug-tdep.c
index 568c2ea..cbde877 100644
--- a/gdb/alpha-mdebug-tdep.c
+++ b/gdb/alpha-mdebug-tdep.c
@@ -136,6 +136,15 @@ find_proc_desc (CORE_ADDR pc)
   return proc_desc;
 }
 
+/* Return a non-null result if the function is frameless; zero otherwise.  */
+
+static int
+alpha_mdebug_frameless (struct mdebug_extra_func_info *proc_desc)
+{
+  return PROC_FRAME_REG (proc_desc) == ALPHA_SP_REGNUM
+    && PROC_FRAME_OFFSET (proc_desc) == 0;
+}
+
 /* This returns the PC of the first inst after the prologue.  If we can't
    find the prologue, then return 0.  */
 
@@ -146,8 +155,7 @@ alpha_mdebug_after_prologue (CORE_ADDR pc, struct mdebug_extra_func_info *proc_d
     {
       /* If function is frameless, then we need to do it the hard way.  I
          strongly suspect that frameless always means prologueless... */
-      if (PROC_FRAME_REG (proc_desc) == ALPHA_SP_REGNUM
-	  && PROC_FRAME_OFFSET (proc_desc) == 0)
+      if (alpha_mdebug_frameless (proc_desc))
 	return 0;
     }
 
@@ -283,6 +291,20 @@ alpha_mdebug_frame_prev_register (struct frame_info *this_frame,
   return trad_frame_get_prev_register (this_frame, info->saved_regs, regnum);
 }
 
+/* Return a non-null result if the size of the stack frame exceeds the
+   maximum debuggable frame size (512 Kbytes); zero otherwise.  */
+
+static int
+alpha_mdebug_max_frame_size_exceeded (struct mdebug_extra_func_info *proc_desc)
+{
+  /* If frame offset is null, we can be in two cases: either the
+     function is frameless (the stack frame is null) or its
+     frame exceeds the maximum debuggable frame size (512 Kbytes).  */
+
+  return PROC_FRAME_OFFSET (proc_desc) == 0
+    && !alpha_mdebug_frameless (proc_desc);
+}
+
 static int
 alpha_mdebug_frame_sniffer (const struct frame_unwind *self,
                             struct frame_info *this_frame,
@@ -302,6 +324,11 @@ alpha_mdebug_frame_sniffer (const struct frame_unwind *self,
   if (alpha_mdebug_in_prologue (pc, proc_desc))
     return 0;
 
+  /* If the maximum debuggable frame size has been exceeded, the
+     proc desc is bogus.  Fall back on the heuristic unwinder.  */
+  if (alpha_mdebug_max_frame_size_exceeded (proc_desc))
+    return 0;
+
   return 1;
 }
 
@@ -362,6 +389,11 @@ alpha_mdebug_frame_base_sniffer (struct frame_info *this_frame)
   if (proc_desc == NULL)
     return NULL;
 
+  /* If the maximum debuggable frame size has been exceeded, the
+     proc desc is bogus.  Fall back on the heuristic unwinder.  */
+  if (alpha_mdebug_max_frame_size_exceeded (proc_desc))
+    return 0;
+
   return &alpha_mdebug_frame_base;
 }
 
diff --git a/gdb/alpha-tdep.c b/gdb/alpha-tdep.c
index 2f583e9..c28a36f 100644
--- a/gdb/alpha-tdep.c
+++ b/gdb/alpha-tdep.c
@@ -46,6 +46,35 @@
 
 #include "alpha-tdep.h"
 
+/* Instruction decoding. The notations for registers, immediates and opcodes
+   are the same as the one used in Compaq's Alpha architecture handbook.  */
+
+#define INSN_OPCODE(insn) ((insn & 0xfc000000) >> 26)
+
+/* Memory instruction format */
+#define MEM_RA(insn) ((insn & 0x03e00000) >> 21)
+#define MEM_RB(insn) ((insn & 0x001f0000) >> 16)
+#define MEM_DISP(insn) \
+  (((insn & 0x8000) == 0) ? (insn & 0xffff) : -((-insn) & 0xffff))
+
+static const int lda_opcode = 0x08;
+static const int stq_opcode = 0x2d;
+
+/* Branch instruction format */
+#define BR_RA(insn) MEM_RA(insn)
+
+static const int bne_opcode = 0x3d;
+
+/* Operate instruction format */
+#define OPR_FUNCTION(insn) ((insn & 0xfe0) >> 5)
+#define OPR_HAS_IMMEDIATE(insn) ((insn & 0x1000) == 0x1000)
+#define OPR_RA(insn) MEM_RA(insn)
+#define OPR_RC(insn) ((insn & 0x1f))
+#define OPR_LIT(insn) ((insn & 0x1fe000) >> 13)
+
+static const int subq_opcode = 0x10;
+static const int subq_function = 0x29;
+
 \f
 /* Return the name of the REGNO register.
 
@@ -1000,6 +1029,108 @@ struct alpha_heuristic_unwind_cache
   int return_reg;
 };
 
+/* If a probing loop sequence starts at PC, simulate it and compute
+   FRAME_SIZE and PC after its execution.  Otherwise, return with PC and
+   FRAME_SIZE unchanged.  */
+
+static void
+alpha_heuristic_analyze_probing_loop (struct gdbarch *gdbarch, CORE_ADDR *pc,
+				      int *frame_size)
+{
+  CORE_ADDR cur_pc = *pc;
+  int cur_frame_size = *frame_size;
+  int nb_of_iterations, reg_index, reg_probe;
+  unsigned int insn;
+
+  /* The following pattern is recognized as a probing loop:
+
+        lda     REG_INDEX,NB_OF_ITERATIONS
+        lda     REG_PROBE,<immediate>(sp)
+
+     LOOP_START:
+        stq     zero,<immediate>(REG_PROBE)
+        subq    REG_INDEX,0x1,REG_INDEX
+        lda     REG_PROBE,<immediate>(REG_PROBE)
+        bne     REG_INDEX, LOOP_START
+ 
+        lda     sp,<immediate>(REG_PROBE)
+
+     If anything different is found, the function returns without
+     changing PC and FRAME_SIZE. Otherwise, PC will point immediately
+     after this sequence, and FRAME_SIZE will be updated.
+  */
+
+  /* lda     REG_INDEX,NB_OF_ITERATIONS */
+
+  insn = alpha_read_insn (gdbarch, cur_pc);
+  if (INSN_OPCODE (insn) != lda_opcode)
+    return;
+  reg_index = MEM_RA (insn);
+  nb_of_iterations = MEM_DISP (insn);
+
+  /* lda     REG_PROBE,<immediate>(sp) */
+
+  cur_pc += ALPHA_INSN_SIZE;
+  insn = alpha_read_insn (gdbarch, cur_pc);
+  if (INSN_OPCODE (insn) != lda_opcode
+      || MEM_RB (insn) != ALPHA_SP_REGNUM)
+    return;
+  reg_probe = MEM_RA (insn);
+  cur_frame_size -= MEM_DISP (insn);
+
+  /* stq     zero,<immediate>(REG_PROBE) */
+  
+  cur_pc += ALPHA_INSN_SIZE;
+  insn = alpha_read_insn (gdbarch, cur_pc);
+  if (INSN_OPCODE (insn) != stq_opcode
+      || MEM_RA (insn) != 0x1f
+      || MEM_RB (insn) != reg_probe)
+    return;
+  
+  /* subq    REG_INDEX,0x1,REG_INDEX */
+
+  cur_pc += ALPHA_INSN_SIZE;
+  insn = alpha_read_insn (gdbarch, cur_pc);
+  if (INSN_OPCODE (insn) != subq_opcode
+      || !OPR_HAS_IMMEDIATE (insn)
+      || OPR_FUNCTION (insn) != subq_function
+      || OPR_LIT(insn) != 1
+      || OPR_RA (insn) != reg_index
+      || OPR_RC (insn) != reg_index)
+    return;
+  
+  /* lda     REG_PROBE,<immediate>(REG_PROBE) */
+  
+  cur_pc += ALPHA_INSN_SIZE;
+  insn = alpha_read_insn (gdbarch, cur_pc);
+  if (INSN_OPCODE (insn) != lda_opcode
+      || MEM_RA (insn) != reg_probe
+      || MEM_RB (insn) != reg_probe)
+    return;
+  cur_frame_size -= MEM_DISP (insn) * nb_of_iterations;
+
+  /* bne     REG_INDEX, LOOP_START */
+
+  cur_pc += ALPHA_INSN_SIZE;
+  insn = alpha_read_insn (gdbarch, cur_pc);
+  if (INSN_OPCODE (insn) != bne_opcode
+      || MEM_RA (insn) != reg_index)
+    return;
+
+  /* lda     sp,<immediate>(REG_PROBE) */
+
+  cur_pc += ALPHA_INSN_SIZE;
+  insn = alpha_read_insn (gdbarch, cur_pc);
+  if (INSN_OPCODE (insn) != lda_opcode
+      || MEM_RA (insn) != ALPHA_SP_REGNUM
+      || MEM_RB (insn) != reg_probe)
+    return;
+  cur_frame_size -= MEM_DISP (insn);
+
+  *pc = cur_pc;
+  *frame_size = cur_frame_size;
+}
+
 static struct alpha_heuristic_unwind_cache *
 alpha_heuristic_frame_unwind_cache (struct frame_info *this_frame,
 				    void **this_prologue_cache,
@@ -1116,6 +1247,8 @@ alpha_heuristic_frame_unwind_cache (struct frame_info *this_frame,
 	    frame_reg = ALPHA_GCC_FP_REGNUM;
 	  else if (word == 0x47fe040f)			/* bis zero,sp,fp */
 	    frame_reg = ALPHA_GCC_FP_REGNUM;
+
+	  alpha_heuristic_analyze_probing_loop (gdbarch, &cur_pc, &frame_size);
 	}
 
       /* If we haven't found a valid return address register yet, keep
-- 
1.6.5.rc2