[PATCH] Improve intel IBT support

[Victor Collod <vcollod@nvidia.com>]

Refactor amd64_analyze_prologue to be more linear, add i386 support for endbr32.

2020-03-12  Victor Collod  <vcollod@nvidia.com>

	* i386-tdep.c (i386_skip_endbr): add a helper function to skip endbr
	instructions.
	(i386_analyze_prologue): call i386_skip_endbr.
	* amd64-tdep.c (amd64_analyze_prologue): make the function more linear
---
 gdb/amd64-tdep.c | 76 +++++++++++++++++++++++-------------------------
 gdb/i386-tdep.c  | 19 ++++++++++++
 2 files changed, 56 insertions(+), 39 deletions(-)

diff --git a/gdb/amd64-tdep.c b/gdb/amd64-tdep.c
index f96a9868259..06d0fe9a194 100644
--- a/gdb/amd64-tdep.c
+++ b/gdb/amd64-tdep.c
@@ -2374,7 +2374,6 @@ amd64_analyze_prologue (struct gdbarch *gdbarch,
 			CORE_ADDR pc, CORE_ADDR current_pc,
 			struct amd64_frame_cache *cache)
 {
-  enum bfd_endian byte_order = gdbarch_byte_order (gdbarch);
   /* The `endbr64` instruction.  */
   static const gdb_byte endbr64[4] = { 0xf3, 0x0f, 0x1e, 0xfa };
   /* There are two variations of movq %rsp, %rbp.  */
@@ -2384,8 +2383,7 @@ amd64_analyze_prologue (struct gdbarch *gdbarch,
   static const gdb_byte mov_esp_ebp_1[2] = { 0x89, 0xe5 };
   static const gdb_byte mov_esp_ebp_2[2] = { 0x8b, 0xec };
 
-  gdb_byte buf[3];
-  gdb_byte op;
+  gdb_byte buf[4];
 
   if (current_pc <= pc)
     return current_pc;
@@ -2395,57 +2393,57 @@ amd64_analyze_prologue (struct gdbarch *gdbarch,
   else
     pc = amd64_analyze_stack_align (pc, current_pc, cache);
 
-  op = read_code_unsigned_integer (pc, 1, byte_order);
+  read_code (pc, buf, 4);
 
   /* Check for the `endbr64` instruction, skip it if found.  */
-  if (op == endbr64[0])
+  if (memcmp (buf, endbr64, sizeof(endbr64)) == 0)
     {
-      read_code (pc + 1, buf, 3);
+      pc += sizeof(endbr64);
+      /* If that's all, return now.  */
+      if (current_pc <= pc)
+	return current_pc;
+      /* update the code buffer, as pc changed */
+      read_code (pc, buf, 1);
+    }
 
-      if (memcmp (buf, &endbr64[1], 3) == 0)
-	pc += 4;
+  /* stop right now if there's no `pushq %rbp' */
+  if (buf[0] != 0x55)
+    return pc;
 
-      op = read_code_unsigned_integer (pc, 1, byte_order);
-    }
+  /* Take into account that we've executed the `pushq %rbp' that
+     starts this instruction sequence.  */
+  cache->saved_regs[AMD64_RBP_REGNUM] = 0;
+  cache->sp_offset += 8;
 
+  pc += 1;
+
+  /* If that's all, return now.  */
   if (current_pc <= pc)
     return current_pc;
 
-  if (op == 0x55)		/* pushq %rbp */
-    {
-      /* Take into account that we've executed the `pushq %rbp' that
-         starts this instruction sequence.  */
-      cache->saved_regs[AMD64_RBP_REGNUM] = 0;
-      cache->sp_offset += 8;
+  read_code (pc, buf, 3);
 
-      /* If that's all, return now.  */
-      if (current_pc <= pc + 1)
-        return current_pc;
-
-      read_code (pc + 1, buf, 3);
+  /* Check for `movq %rsp, %rbp'.  */
+  if (memcmp (buf, mov_rsp_rbp_1, 3) == 0
+      || memcmp (buf, mov_rsp_rbp_2, 3) == 0)
+    {
+      pc += 3;
+      /* OK, we actually have a frame.  */
+      cache->frameless_p = 0;
+      return pc;
+    }
 
-      /* Check for `movq %rsp, %rbp'.  */
-      if (memcmp (buf, mov_rsp_rbp_1, 3) == 0
-	  || memcmp (buf, mov_rsp_rbp_2, 3) == 0)
+  /* For X32, also check for `movl %esp, %ebp'.  */
+  if (gdbarch_ptr_bit (gdbarch) == 32)
+    {
+      if (memcmp (buf, mov_esp_ebp_1, 2) == 0
+	  || memcmp (buf, mov_esp_ebp_2, 2) == 0)
 	{
+	  pc += 2;
 	  /* OK, we actually have a frame.  */
 	  cache->frameless_p = 0;
-	  return pc + 4;
+	  return pc;
 	}
-
-      /* For X32, also check for `movq %esp, %ebp'.  */
-      if (gdbarch_ptr_bit (gdbarch) == 32)
-	{
-	  if (memcmp (buf, mov_esp_ebp_1, 2) == 0
-	      || memcmp (buf, mov_esp_ebp_2, 2) == 0)
-	    {
-	      /* OK, we actually have a frame.  */
-	      cache->frameless_p = 0;
-	      return pc + 3;
-	    }
-	}
-
-      return pc + 1;
     }
 
   return pc;
diff --git a/gdb/i386-tdep.c b/gdb/i386-tdep.c
index e87d7f36356..f7670a7febb 100644
--- a/gdb/i386-tdep.c
+++ b/gdb/i386-tdep.c
@@ -1537,6 +1537,24 @@ struct i386_insn i386_frame_setup_skip_insns[] =
   { 0 }
 };
 
+/* Check whether PC points to an endbr32 instruction.  */
+static CORE_ADDR
+i386_skip_endbr(CORE_ADDR pc)
+{
+  static const gdb_byte endbr32[] = { 0xf3, 0x0f, 0x1e, 0xfb };
+
+  gdb_byte buf[sizeof(endbr32)];
+
+  /* Stop there if we can't read the code */
+  if (target_read_code (pc, buf, sizeof(endbr32)))
+    return pc;
+
+  /* If the instruction isn't an endbr32, stop */
+  if (memcmp (buf, endbr32, sizeof(endbr32)) != 0)
+    return pc;
+
+  return pc + sizeof(endbr32);
+}
 
 /* Check whether PC points to a no-op instruction.  */
 static CORE_ADDR
@@ -1814,6 +1832,7 @@ i386_analyze_prologue (struct gdbarch *gdbarch,
 		       CORE_ADDR pc, CORE_ADDR current_pc,
 		       struct i386_frame_cache *cache)
 {
+  pc = i386_skip_endbr (pc);
   pc = i386_skip_noop (pc);
   pc = i386_follow_jump (gdbarch, pc);
   pc = i386_analyze_struct_return (pc, current_pc, cache);
-- 
2.20.1