Re: ptrace(2) autoconf tests

[Mark Kettenis <kettenis@chello.nl>]

   Date: Sun, 08 Aug 2004 17:42:16 -0400
   From: Michael Chastain <mec.gnu@mindspring.com>

   Here's all the ptrace(2) prototypes I found on hp testdrive machines.

Ah, smart.  I signed myself up again.  The missing prototype on Tru64
is suspicious; is the return value really `int'?  Current GDB code
seems to suggest it isn't.

Anyway, below is a first hack at the autoconf tests.  The tests
themselves are inspired by the standard AC_FUNC_CLOSEDIR_VOID and
AC_FUNC_SELECT_ARGTYPES tests.  My code defines PTRACE_LONG if the
return type of ptrace(2) is `long', defines PTRACE_TYPE_ARG3 to the
type of the third argument of ptrace(2) and defines PTRACE_TYPE_ARG5
to the type of the fifth argument of ptrace(2) if it has one.
PTRACE_TYPE_ARG3 will replace PTRACE_ARG3_TYPE.  PTRACE_XFER_TYPE will
be set based on PTRACE_LONG and PTRACE_TYPE_ARG5 will replace
FIVE_ARG_PTRACE.

I've verified that it gets things right on FreeBSD/i386 4.7, HP-UX
11.00, HP-UX 11.11, HP-UX 11.23, RedHat 7.2 Alpha, Tru64 4.0g, Tru64
5.1b, Ultrix/VAX 4.0 and Solaris 2.9 SPARC.  If possible I've also
tried the vendor compiler.  The bundled compiler on HP-UX
(/usr/bin/cc) gets things wrong, but that's the broken K&R compiler
that cannot compile GDB anyway.

If we all agree this is a good thing, that is, if nobody objetcs) I'll
probably integrate and commit this next weekend or so.

Mark



dnl Process this file with autoconf to produce a configure script.
AC_INIT(configure.in)
AC_PREREQ(2.13)
AC_CONFIG_HEADER(config.h)

dnl Checks for programs.
AC_PROG_CC
AC_AIX

dnl Checks for libraries.

dnl Checks for header files.
AC_CHECK_HEADERS(sys/types.h)
AC_CHECK_HEADERS(sys/ptrace.h unistd.h)

dnl Checks for typedefs, structures, and compiler characteristics.

dnl Checks for library functions.
gdb_ptrace_headers='
#if HAVE_SYS_TYPES_H
# include <sys/types.h>
#endif
#if HAVE_SYS_PTRACE_H
# include <sys/ptrace.h>
#endif
#if HAVE_UNISTD_H
# include <unistd.h>
#endif
'

# XXX Replace with AC_CHECK_DECL.
AC_CACHE_CHECK([whether ptrace is declared], gdb_cv_have_decl_ptrace,
  AC_TRY_COMPILE($gdb_ptrace_headers, [
#ifndef ptrace
  char *p = (char *) ptrace;
#endif
],
    gdb_cv_have_decl_ptrace=yes,
    gdb_cv_have_decl_ptrace=no))
if test $gdb_cv_have_decl_ptrace = no; then
  : ${gdb_cv_func_ptrace_long=no}
  : ${gdb_cv_func_ptrace_args='int,int,long,long'}
fi

AC_CACHE_CHECK([whether ptrace returns long], gdb_cv_func_ptrace_long,
  AC_TRY_COMPILE($gdb_ptrace_headers,
    [extern long ptrace ();],
    gdb_cv_func_ptrace_long=yes,
    gdb_cv_func_ptrace_long=no))
if test $gdb_cv_func_ptrace_long = yes; then
  AC_DEFINE(PTRACE_LONG, 1,
            [Define to 1 if the ptrace function returns long instead of int.])
fi

AC_CACHE_CHECK([types of arguments for ptrace], gdb_cv_func_ptrace_args, [
if test "$gdb_cv_func_ptrace_long" = yes; then
  gdb_ret='long'
else
  gdb_ret='int'
fi
for gdb_arg1 in 'int' 'long'; do
 for gdb_arg2 in 'pid_t' 'int' 'long'; do
  for gdb_arg3 in 'int *' 'caddr_t' 'int' 'long'; do
   for gdb_arg4 in 'int' 'long'; do
     AC_TRY_COMPILE($gdb_ptrace_headers, [
extern $gdb_ret ptrace ($gdb_arg1, $gdb_arg2, $gdb_arg3, $gdb_arg4);
], [gdb_cv_func_ptrace_args="$gdb_arg1,$gdb_arg2,$gdb_arg3,$gdb_arg4";
    break 4;])
    for gdb_arg5 in 'int *' 'int' 'long'; do
     AC_TRY_COMPILE($gdb_ptrace_headers, [
extern $gdb_ret ptrace ($gdb_arg1, $gdb_arg2, $gdb_arg3, $gdb_arg4, $gdb_arg5);
], [
gdb_cv_func_ptrace_args="$gdb_arg1,$gdb_arg2,$gdb_arg3,$gdb_arg4,$gdb_arg5";
    break 5;])
    done
   done
  done
 done
done
# Provide a safe default value.
: ${gdb_cv_func_ptrace_args='int,int,long,long'}
])
ac_save_IFS=$IFS; IFS=','
set dummy `echo "$gdb_cv_func_ptrace_args" | sed 's/\*/\*/g'`
IFS=$ac_save_IFS
shift
AC_DEFINE_UNQUOTED(PTRACE_TYPE_ARG3, ($[3]),
		   [Define to the type of arg 3 for ptrace.])
if test -n "$[5]"; then
  AC_DEFINE_UNQUOTED(PTRACE_TYPE_ARG5, ($[5]),
		     [Define to the type of arg 5 for ptrace.])
fi

AC_OUTPUT()