[PATCH v2 2/2] Refactor amd64_analyze_prologue

[Victor Collod <vcollod@nvidia.com>]

* merge op and the buf array, which were both used for storing code
* invert conditions to avoid long nested ifs
* use target_read_code instead of read_code to gracefully handle errors
* `if (current_pc <= pc)' felt backwards, as current_pc doesn't change,
  and the test could be described as "stop if pc went past current_pc"

2020-06-11  Victor Collod  <vcollod@nvidia.com>

	* amd64-tdep.c (amd64_analyze_prologue): Make the function more readable.
---
 gdb/amd64-tdep.c | 93 ++++++++++++++++++++++++++----------------------
 1 file changed, 50 insertions(+), 43 deletions(-)

diff --git a/gdb/amd64-tdep.c b/gdb/amd64-tdep.c
index 0ce9fbc2997..6c1a4a138de 100644
--- a/gdb/amd64-tdep.c
+++ b/gdb/amd64-tdep.c
@@ -2374,7 +2374,6 @@ amd64_analyze_prologue (struct gdbarch *gdbarch,
 			CORE_ADDR pc, CORE_ADDR current_pc,
 			struct amd64_frame_cache *cache)
 {
-  enum bfd_endian byte_order = gdbarch_byte_order (gdbarch);
   /* The `endbr64` instruction.  */
   static const gdb_byte endbr64[4] = { 0xf3, 0x0f, 0x1e, 0xfa };
   /* There are two variations of movq %rsp, %rbp.  */
@@ -2384,10 +2383,10 @@ amd64_analyze_prologue (struct gdbarch *gdbarch,
   static const gdb_byte mov_esp_ebp_1[2] = { 0x89, 0xe5 };
   static const gdb_byte mov_esp_ebp_2[2] = { 0x8b, 0xec };
 
-  gdb_byte buf[3];
-  gdb_byte op;
+  gdb_byte buf[4];
 
-  if (current_pc <= pc)
+  /* Analysis must not go past current_pc */
+  if (pc >= current_pc)
     return current_pc;
 
   if (gdbarch_ptr_bit (gdbarch) == 32)
@@ -2395,57 +2394,65 @@ amd64_analyze_prologue (struct gdbarch *gdbarch,
   else
     pc = amd64_analyze_stack_align (pc, current_pc, cache);
 
-  op = read_code_unsigned_integer (pc, 1, byte_order);
-
-  /* Check for the `endbr64` instruction, skip it if found.  */
-  if (op == endbr64[0])
+  /* Try to read enough bytes to check for `endbr64'  */
+  if (target_read_code (pc, buf, 4) != 0)
+    {
+      /* If it fails, read just enough data for `pushq %rbp' */
+      if (target_read_code (pc, buf, 1) != 0)
+	return pc;
+    }
+  /* If reading succeeded, check for the `endbr64' instruction and skip it if found.  */
+  else if (memcmp (buf, endbr64, sizeof (endbr64)) == 0)
     {
-      read_code (pc + 1, buf, 3);
+      pc += sizeof (endbr64);
+      /* If we went past the allowed bound, stop */
+      if (pc >= current_pc)
+	return current_pc;
+      /* update the code buffer, as pc changed */
+      if (target_read_code (pc, buf, 1) != 0)
+	return pc;
+    }
 
-      if (memcmp (buf, &endbr64[1], 3) == 0)
-	pc += 4;
+  /* stop right now if there's no `pushq %rbp' */
+  if (buf[0] != 0x55)
+    return pc;
 
-      op = read_code_unsigned_integer (pc, 1, byte_order);
-    }
+  /* Take into account that we've executed the `pushq %rbp' that
+     starts this instruction sequence.  */
+  cache->saved_regs[AMD64_RBP_REGNUM] = 0;
+  cache->sp_offset += 8;
 
-  if (current_pc <= pc)
-    return current_pc;
+  pc += 1;
 
-  if (op == 0x55)		/* pushq %rbp */
-    {
-      /* Take into account that we've executed the `pushq %rbp' that
-         starts this instruction sequence.  */
-      cache->saved_regs[AMD64_RBP_REGNUM] = 0;
-      cache->sp_offset += 8;
+  /* If we went past the allowed bound, stop */
+  if (pc >= current_pc)
+    return current_pc;
 
-      /* If that's all, return now.  */
-      if (current_pc <= pc + 1)
-        return current_pc;
+  /* Try to read the code for the stack base move */
+  if (target_read_code (pc, buf, 3) != 0)
+    return pc;
 
-      read_code (pc + 1, buf, 3);
+  /* Check for `movq %rsp, %rbp'.  */
+  if (memcmp (buf, mov_rsp_rbp_1, 3) == 0
+      || memcmp (buf, mov_rsp_rbp_2, 3) == 0)
+    {
+      pc += 3;
+      /* OK, we actually have a frame.  */
+      cache->frameless_p = 0;
+      return pc;
+    }
 
-      /* Check for `movq %rsp, %rbp'.  */
-      if (memcmp (buf, mov_rsp_rbp_1, 3) == 0
-	  || memcmp (buf, mov_rsp_rbp_2, 3) == 0)
+  /* For X32, also check for `movl %esp, %ebp'.  */
+  if (gdbarch_ptr_bit (gdbarch) == 32)
+    {
+      if (memcmp (buf, mov_esp_ebp_1, 2) == 0
+	  || memcmp (buf, mov_esp_ebp_2, 2) == 0)
 	{
+	  pc += 2;
 	  /* OK, we actually have a frame.  */
 	  cache->frameless_p = 0;
-	  return pc + 4;
+	  return pc;
 	}
-
-      /* For X32, also check for `movq %esp, %ebp'.  */
-      if (gdbarch_ptr_bit (gdbarch) == 32)
-	{
-	  if (memcmp (buf, mov_esp_ebp_1, 2) == 0
-	      || memcmp (buf, mov_esp_ebp_2, 2) == 0)
-	    {
-	      /* OK, we actually have a frame.  */
-	      cache->frameless_p = 0;
-	      return pc + 3;
-	    }
-	}
-
-      return pc + 1;
     }
 
   return pc;
-- 
2.20.1