[rfc, gdbserver] Disable address space randomization

["Ulrich Weigand" <uweigand@de.ibm.com>]

Hello,

this patch disables address space randomization on Linux in gdbserver.
The implementation is pretty much copied from GDB's linux-nat.c.

At this point this happens unconditionally, whenever the kernel
supports the personality system call.  If necessary, it would
be possible to make this configurable by adding a command line
argument to gdbserver ...

Tested on i386-linux.  Fixes a couple of test failures on Ubuntu.

Any thoughts or comments?

Bye,
Ulrich


ChangeLog:

	* configure.ac: Check support for personality routine.
	* configure: Regenerate.
	* config.in: Likewise.
	* linux-low.c (linux_create_inferior): Disable address space
	randomization when forking inferior.


Index: gdb/gdbserver/config.in
===================================================================
RCS file: /cvs/src/src/gdb/gdbserver/config.in,v
retrieving revision 1.32
diff -u -p -r1.32 config.in
--- gdb/gdbserver/config.in	21 Jul 2011 23:46:12 -0000	1.32
+++ gdb/gdbserver/config.in	21 Sep 2011 14:06:31 -0000
@@ -18,6 +18,10 @@
 /* Define to 1 if you have the <arpa/inet.h> header file. */
 #undef HAVE_ARPA_INET_H
 
+/* Define to 1 if you have the declaration of `ADDR_NO_RANDOMIZE', and to 0 if
+   you don't. */
+#undef HAVE_DECL_ADDR_NO_RANDOMIZE
+
 /* Define to 1 if you have the declaration of `memmem', and to 0 if you don't.
    */
 #undef HAVE_DECL_MEMMEM
@@ -96,6 +100,9 @@
 /* Define to 1 if you have the <netinet/tcp.h> header file. */
 #undef HAVE_NETINET_TCP_H
 
+/* Define if you support the personality syscall. */
+#undef HAVE_PERSONALITY
+
 /* Define to 1 if you have the `pread' function. */
 #undef HAVE_PREAD
 
Index: gdb/gdbserver/configure
===================================================================
RCS file: /cvs/src/src/gdb/gdbserver/configure,v
retrieving revision 1.60
diff -u -p -r1.60 configure
--- gdb/gdbserver/configure	21 Jul 2011 23:46:12 -0000	1.60
+++ gdb/gdbserver/configure	21 Sep 2011 14:06:32 -0000
@@ -5066,6 +5066,80 @@ fi
 rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
 CFLAGS="$saved_cflags"
 
+ac_fn_c_check_decl "$LINENO" "ADDR_NO_RANDOMIZE" "ac_cv_have_decl_ADDR_NO_RANDOMIZE" "#include <sys/personality.h>
+"
+if test "x$ac_cv_have_decl_ADDR_NO_RANDOMIZE" = x""yes; then :
+  ac_have_decl=1
+else
+  ac_have_decl=0
+fi
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_DECL_ADDR_NO_RANDOMIZE $ac_have_decl
+_ACEOF
+
+
+if test "$cross_compiling" = yes; then :
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+#include <sys/personality.h>
+int
+main ()
+{
+
+#      if !HAVE_DECL_ADDR_NO_RANDOMIZE
+#       define ADDR_NO_RANDOMIZE 0x0040000
+#      endif
+       /* Test the flag could be set and stays set.  */
+       personality (personality (0xffffffff) | ADDR_NO_RANDOMIZE);
+       if (!(personality (personality (0xffffffff)) & ADDR_NO_RANDOMIZE))
+           return 1
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+  gdbsrv_cv_have_personality=true
+else
+  gdbsrv_cv_have_personality=false
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext conftest.$ac_ext
+else
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+#include <sys/personality.h>
+int
+main ()
+{
+
+#      if !HAVE_DECL_ADDR_NO_RANDOMIZE
+#       define ADDR_NO_RANDOMIZE 0x0040000
+#      endif
+       /* Test the flag could be set and stays set.  */
+       personality (personality (0xffffffff) | ADDR_NO_RANDOMIZE);
+       if (!(personality (personality (0xffffffff)) & ADDR_NO_RANDOMIZE))
+           return 1
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_run "$LINENO"; then :
+  gdbsrv_cv_have_personality=true
+else
+  gdbsrv_cv_have_personality=false
+fi
+rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \
+  conftest.$ac_objext conftest.beam conftest.$ac_ext
+fi
+
+if $gdbsrv_cv_have_personality
+then
+
+$as_echo "#define HAVE_PERSONALITY 1" >>confdefs.h
+
+fi
+
 
 IPA_DEPFILES=""
 extra_libraries=""
Index: gdb/gdbserver/configure.ac
===================================================================
RCS file: /cvs/src/src/gdb/gdbserver/configure.ac,v
retrieving revision 1.47
diff -u -p -r1.47 configure.ac
--- gdb/gdbserver/configure.ac	21 Jul 2011 23:46:12 -0000	1.47
+++ gdb/gdbserver/configure.ac	21 Sep 2011 14:06:32 -0000
@@ -358,6 +358,29 @@ AC_COMPILE_IFELSE(AC_LANG_PROGRAM([]),
 	        	[gdbsrv_cv_have_visibility_hidden=no])
 CFLAGS="$saved_cflags"
 
+dnl Check if we can disable the virtual address space randomization.
+dnl The functionality of setarch -R.
+AC_CHECK_DECLS([ADDR_NO_RANDOMIZE],,, [#include <sys/personality.h>])
+define([PERSONALITY_TEST], [AC_LANG_PROGRAM([#include <sys/personality.h>], [
+#      if !HAVE_DECL_ADDR_NO_RANDOMIZE
+#       define ADDR_NO_RANDOMIZE 0x0040000
+#      endif
+       /* Test the flag could be set and stays set.  */
+       personality (personality (0xffffffff) | ADDR_NO_RANDOMIZE);
+       if (!(personality (personality (0xffffffff)) & ADDR_NO_RANDOMIZE))
+           return 1])])
+AC_RUN_IFELSE([PERSONALITY_TEST],
+              [gdbsrv_cv_have_personality=true],
+              [gdbsrv_cv_have_personality=false],
+              [AC_LINK_IFELSE([PERSONALITY_TEST],
+                              [gdbsrv_cv_have_personality=true],
+                              [gdbsrv_cv_have_personality=false])])
+if $gdbsrv_cv_have_personality
+then
+    AC_DEFINE([HAVE_PERSONALITY], 1,
+              [Define if you support the personality syscall.])
+fi
+
 
 IPA_DEPFILES=""
 extra_libraries=""
Index: gdb/gdbserver/linux-low.c
===================================================================
RCS file: /cvs/src/src/gdb/gdbserver/linux-low.c,v
retrieving revision 1.176
diff -u -p -r1.176 linux-low.c
--- gdb/gdbserver/linux-low.c	24 Aug 2011 12:17:39 -0000	1.176
+++ gdb/gdbserver/linux-low.c	21 Sep 2011 14:06:33 -0000
@@ -55,6 +55,13 @@
 #define SPUFS_MAGIC 0x23c9b64e
 #endif
 
+#ifdef HAVE_PERSONALITY
+# include <sys/personality.h>
+# if !HAVE_DECL_ADDR_NO_RANDOMIZE
+#  define ADDR_NO_RANDOMIZE 0x0040000
+# endif
+#endif
+
 #ifndef O_LARGEFILE
 #define O_LARGEFILE 0
 #endif
@@ -520,10 +527,27 @@ add_lwp (ptid_t ptid)
 static int
 linux_create_inferior (char *program, char **allargs)
 {
+#ifdef HAVE_PERSONALITY
+  int personality_orig = 0, personality_set = 0;
+#endif
   struct lwp_info *new_lwp;
   int pid;
   ptid_t ptid;
 
+#ifdef HAVE_PERSONALITY
+  errno = 0;
+  personality_orig = personality (0xffffffff);
+  if (errno == 0 && !(personality_orig & ADDR_NO_RANDOMIZE))
+    {
+      personality_set = 1;
+      personality (personality_orig | ADDR_NO_RANDOMIZE);
+    }
+  if (errno != 0 || (personality_set
+		     && !(personality (0xffffffff) & ADDR_NO_RANDOMIZE)))
+    warning ("Error disabling address space randomization: %s",
+	     strerror (errno));
+#endif
+
 #if defined(__UCLIBC__) && defined(HAS_NOMMU)
   pid = vfork ();
 #else
@@ -552,6 +576,17 @@ linux_create_inferior (char *program, ch
       _exit (0177);
     }
 
+#ifdef HAVE_PERSONALITY
+  if (personality_set)
+    {
+      errno = 0;
+      personality (personality_orig);
+      if (errno != 0)
+	warning ("Error restoring address space randomization: %s",
+		 strerror (errno));
+    }
+#endif
+
   linux_add_process (pid, 0);
 
   ptid = ptid_build (pid, pid, 0);

-- 
  Dr. Ulrich Weigand
  GNU Toolchain for Linux on System z and Cell BE
  Ulrich.Weigand@de.ibm.com