PATCH: PR gdb/13969: GDBserver doesn't check unsupported binary

["H.J. Lu" <hongjiu.lu@intel.com>]

Hi,

This patch checks if a binary is supported by GDBserver.  Also checking
ELF64 isn't very reliable to detect 64bit register support since x32
is ELF32, but it does support 64bit register.  This patch checks the
e_machine field for 64bit register support.  OK for trunk?

Thanks.


H.J.
---
2012-04-11  H.J. Lu  <hongjiu.lu@intel.com>

	PR gdb/13969
	* linux-low.c (linux_pid_exe_is_elf_64_file): Also return the
	e_machine field.
	(linux_qxfer_libraries_svr4): Update call to elf_64_file_p.

	* linux-low.h (linux_pid_exe_is_elf_64_file): Updated.

	* linux-x86-low.c (linux_is_64bit): New.
	(x86_arch_setup): Check if GDBserver is compatible with
	process.

diff --git a/gdb/gdbserver/linux-low.c b/gdb/gdbserver/linux-low.c
index f7a83f4..35d7e69 100644
--- a/gdb/gdbserver/linux-low.c
+++ b/gdb/gdbserver/linux-low.c
@@ -263,13 +263,19 @@ static void wait_for_sigstop (struct inferior_list_entry *entry);
 /* Return non-zero if HEADER is a 64-bit ELF file.  */
 
 static int
-elf_64_header_p (const Elf64_Ehdr *header)
+elf_64_header_p (const Elf64_Ehdr *header, unsigned int *machine)
 {
-  return (header->e_ident[EI_MAG0] == ELFMAG0
-          && header->e_ident[EI_MAG1] == ELFMAG1
-          && header->e_ident[EI_MAG2] == ELFMAG2
-          && header->e_ident[EI_MAG3] == ELFMAG3
-          && header->e_ident[EI_CLASS] == ELFCLASS64);
+  if (header->e_ident[EI_MAG0] == ELFMAG0
+      && header->e_ident[EI_MAG1] == ELFMAG1
+      && header->e_ident[EI_MAG2] == ELFMAG2
+      && header->e_ident[EI_MAG3] == ELFMAG3)
+    {
+      *machine = header->e_machine;
+      return header->e_ident[EI_CLASS] == ELFCLASS64;
+
+    }
+  *machine = EM_NONE;
+  return -1;
 }
 
 /* Return non-zero if FILE is a 64-bit ELF file,
@@ -277,7 +283,7 @@ elf_64_header_p (const Elf64_Ehdr *header)
    and -1 if the file is not accessible or doesn't exist.  */
 
 static int
-elf_64_file_p (const char *file)
+elf_64_file_p (const char *file, unsigned int *machine)
 {
   Elf64_Ehdr header;
   int fd;
@@ -293,19 +299,19 @@ elf_64_file_p (const char *file)
     }
   close (fd);
 
-  return elf_64_header_p (&header);
+  return elf_64_header_p (&header, machine);
 }
 
 /* Accepts an integer PID; Returns true if the executable PID is
    running is a 64-bit ELF file..  */
 
 int
-linux_pid_exe_is_elf_64_file (int pid)
+linux_pid_exe_is_elf_64_file (int pid, unsigned int *machine)
 {
   char file[MAXPATHLEN];
 
   sprintf (file, "/proc/%d/exe", pid);
-  return elf_64_file_p (file);
+  return elf_64_file_p (file, machine);
 }
 
 static void
@@ -5584,6 +5590,7 @@ linux_qxfer_libraries_svr4 (const char *annex, unsigned char *readbuf,
       32     /* l_prev offset in link_map.  */
     };
   const struct link_map_offsets *lmo;
+  unsigned int machine;
 
   if (writebuf != NULL)
     return -2;
@@ -5592,7 +5599,7 @@ linux_qxfer_libraries_svr4 (const char *annex, unsigned char *readbuf,
 
   pid = lwpid_of (get_thread_lwp (current_inferior));
   xsnprintf (filename, sizeof filename, "/proc/%d/exe", pid);
-  is_elf64 = elf_64_file_p (filename);
+  is_elf64 = elf_64_file_p (filename, &machine);
   lmo = is_elf64 ? &lmo_64bit_offsets : &lmo_32bit_offsets;
 
   if (priv->r_debug == 0)
diff --git a/gdb/gdbserver/linux-low.h b/gdb/gdbserver/linux-low.h
index 07eda12..a1a6777 100644
--- a/gdb/gdbserver/linux-low.h
+++ b/gdb/gdbserver/linux-low.h
@@ -278,7 +278,7 @@ struct lwp_info
 
 extern struct inferior_list all_lwps;
 
-int linux_pid_exe_is_elf_64_file (int pid);
+int linux_pid_exe_is_elf_64_file (int pid, unsigned int *machine);
 
 void linux_attach_lwp (unsigned long pid);
 struct lwp_info *find_lwp_pid (ptid_t ptid);
diff --git a/gdb/gdbserver/linux-x86-low.c b/gdb/gdbserver/linux-x86-low.c
index 7feb721..8a4ab50 100644
--- a/gdb/gdbserver/linux-x86-low.c
+++ b/gdb/gdbserver/linux-x86-low.c
@@ -1106,17 +1106,28 @@ x86_linux_process_qsupported (const char *query)
 static void
 x86_arch_setup (void)
 {
-#ifdef __x86_64__
   int pid = pid_of (get_thread_lwp (current_inferior));
-  int use_64bit = linux_pid_exe_is_elf_64_file (pid);
+  unsigned int machine;
+  int use_64bit = linux_pid_exe_is_elf_64_file (pid, &machine);
 
+  if (sizeof (void *) == 4)
+    {
+      if (use_64bit > 0)
+	error (_("Can't debug 64-bit process with 32-bit GDBserver"));
+#ifndef __x86_64__
+      else if (machine == EM_X86_64)
+	error (_("Can't debug x86-64 process with 32-bit GDBserver"));
+#endif
+    }
+
+#ifdef __x86_64__
   if (use_64bit < 0)
     {
       /* This can only happen if /proc/<pid>/exe is unreadable,
 	 but "that can't happen" if we've gotten this far.
 	 Fall through and assume this is a 32-bit program.  */
     }
-  else if (use_64bit)
+  else if (machine == EM_X86_64)
     {
       /* Amd64 doesn't have HAVE_LINUX_USRREGS.  */
       the_low_target.num_regs = -1;