Re: (toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

Re: (toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

[Nathanael Nerode]

Alex pointed out:
>This unfortunately means that your fix is not quite perfect yet, since
>multiple transform commands would be word-split and sed might take
>them as input file names.  I suggest replacing:
>
>        t='$(program_transform_name)'; echo as | sed -e $$t ; \
>
>with
>
>        echo as | sed '$(program_transform_name)'; \
>
>that is the construct used by automake.

Unfortunately, I really have to go to sleep now. :-(  And I have to 
focus my mind on other things for a few days, too.

Please feel free to make this change on your own; it's obviously 
correct as far as I'm concerned.

And likewise to anyone else who wants to improve the top level;
now that autoconfiscation, and configure-on-demand, have actually 
happened, I just don't need the tight control on the top level files 
which I was exercising before.  (At least until I start on the new build 
scheme, which won't be for a while.)  

Part of the motivation for autoconfiscating was to make the top level less 
of a "secret region" which only the "initiated" could hack on.  I hope 
I've accomplished that.

--Nathanael

Re: (toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

[Alexandre Oliva]

[-- Attachment #1: Type: text/plain, Size: 357 bytes --]

On Dec 28, 2002, Nathanael Nerode <neroden@twcny.rr.com> wrote:

> Unfortunately, I really have to go to sleep now. :-(

Fair enough.  Hope you had a good night :-)

> Please feel free to make this change on your own; it's obviously 
> correct as far as I'm concerned.

Done.  While at that, I fixed a few other fallouts.  I'm checking in
the patch below.


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: makefiles.patch --]
[-- Type: text/x-patch, Size: 4892 bytes --]

Index: ChangeLog
from  Alexandre Oliva  <aoliva@redhat.com>

	* Makefile.tpl ($(NOTPARALLEL)): Move to the end.  Bring uses of
	program_transform_name to standard idiom.
	(AUTOGEN, AUTOCONF): Define.
	(Makefile.in): Use $(AUTOGEN).
	(Makefile): Depend on config.status, and use autoconf-style rule to
	build it.  Move original commands to...
	(config.status): ... this new target.
	(configure): Add $(srcdir).  Depend on config/acx.m4.  Use
	$(AUTOCONF).
	* Makefile.in: Rebuilt.

Index: Makefile.tpl
===================================================================
RCS file: /cvs/gcc/gcc/Makefile.tpl,v
retrieving revision 1.25
diff -u -p -r1.25 Makefile.tpl
--- Makefile.tpl 28 Dec 2002 09:12:19 -0000 1.25
+++ Makefile.tpl 28 Dec 2002 16:21:54 -0000
@@ -23,12 +23,6 @@ in
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 #
 
-# Tell GNU make 3.79 not to run the top level in parallel.  This 
-# prevents contention for $builddir/$target/config.cache, as well
-# as minimizing scatter in file system caches.
-NOTPARALLEL = .NOTPARALLEL
-$(NOTPARALLEL):
-
 VPATH=@srcdir@
 links=@configlinks@
 
@@ -282,7 +276,7 @@ USUAL_AS_FOR_TARGET = ` \
     if [ '$(host_canonical)' = '$(target_canonical)' ] ; then \
       echo $(AS); \
     else \
-       t='$(program_transform_name)'; echo as | sed -e $$t ; \
+       echo as | sed '$(program_transform_name)' ; \
     fi; \
   fi`
 
@@ -296,7 +290,7 @@ USUAL_LD_FOR_TARGET = ` \
     if [ '$(host_canonical)' = '$(target_canonical)' ] ; then \
       echo $(LD); \
     else \
-       t='$(program_transform_name)'; echo ld | sed -e $$t ; \
+       echo ld | sed '$(program_transform_name)' ; \
     fi; \
   fi`
 
@@ -308,7 +302,7 @@ USUAL_DLLTOOL_FOR_TARGET = ` \
     if [ '$(host_canonical)' = '$(target_canonical)' ] ; then \
       echo $(DLLTOOL); \
     else \
-       t='$(program_transform_name)'; echo dlltool | sed -e $$t ; \
+       echo dlltool | sed '$(program_transform_name)' ; \
     fi; \
   fi`
 
@@ -320,7 +314,7 @@ USUAL_WINDRES_FOR_TARGET = ` \
     if [ '$(host_canonical)' = '$(target_canonical)' ] ; then \
       echo $(WINDRES); \
     else \
-       t='$(program_transform_name)'; echo windres | sed -e $$t ; \
+       echo windres | sed '$(program_transform_name)' ; \
     fi; \
   fi`
 
@@ -332,7 +326,7 @@ USUAL_AR_FOR_TARGET = ` \
     if [ '$(host_canonical)' = '$(target_canonical)' ] ; then \
       echo $(AR); \
     else \
-       t='$(program_transform_name)'; echo ar | sed -e $$t ; \
+       echo ar | sed '$(program_transform_name)' ; \
     fi; \
   fi`
 
@@ -348,7 +342,7 @@ USUAL_RANLIB_FOR_TARGET = ` \
          echo ranlib; \
       fi; \
     else \
-       t='$(program_transform_name)'; echo ranlib | sed -e $$t ; \
+       echo ranlib | sed '$(program_transform_name)' ; \
     fi; \
   fi`
 
@@ -362,7 +356,7 @@ USUAL_NM_FOR_TARGET = ` \
     if [ '$(host_canonical)' = '$(target_canonical)' ] ; then \
       echo $(NM); \
     else \
-       t='$(program_transform_name)'; echo nm | sed -e $$t ; \
+       echo nm | sed '$(program_transform_name)' ; \
     fi; \
   fi`
 
@@ -1414,19 +1408,30 @@ multilib.ts: maybe-all-gcc
 	echo timestamp > multilib.ts
 
 # Rebuilding Makefile.in, using autogen.
+AUTOGEN = autogen
 $(srcdir)/Makefile.in: # $(srcdir)/Makefile.tpl $(srcdir)/Makefile.def
-	cd $(srcdir) && autogen Makefile.def
+	cd $(srcdir) && $(AUTOGEN) Makefile.def
 
 # Rebuilding Makefile.
-Makefile: Makefile.in configure $(gcc_version_trigger)
+Makefile: $(srcdir)/Makefile.in config.status
+	CONFIG_FILES=$@ CONFIG_HEADERS= $(SHELL) ./config.status
+
+config.status: configure $(gcc_version_trigger)
 	$(SHELL) ./config.status --recheck
 
 # Rebuilding configure.
-configure: configure.in
-	cd $(srcdir) && autoconf
+AUTOCONF = autoconf
+$(srcdir)/configure: $(srcdir)/configure.in $(srcdir)/config/acx.m4
+	cd $(srcdir) && $(AUTOCONF)
 #
 
 .NOEXPORT:
 MAKEOVERRIDES=
+
+# Tell GNU make 3.79 not to run the top level in parallel.  This 
+# prevents contention for $builddir/$target/config.cache, as well
+# as minimizing scatter in file system caches.
+NOTPARALLEL = .NOTPARALLEL
+$(NOTPARALLEL):
 
 # end of Makefile.in
Index: contrib/ChangeLog
from  Alexandre Oliva  <aoliva@redhat.com>
	* gcc_update (configure): Depend on config/acx.m4 as well.

Index: contrib/gcc_update
===================================================================
RCS file: /cvs/gcc/gcc/contrib/gcc_update,v
retrieving revision 1.39
diff -u -p -r1.39 gcc_update
--- contrib/gcc_update 28 Dec 2002 13:21:17 -0000 1.39
+++ contrib/gcc_update 28 Dec 2002 16:21:54 -0000
@@ -106,7 +106,7 @@ libjava/libltdl/configure: libjava/liblt
 libjava/libltdl/stamp-h.in: libjava/libltdl/configure.in libjava/libltdl/aclocal.m4 libjava/libltdl/acconfig.h
 # Top level
 Makefile.in: Makefile.tpl Makefile.def
-configure: configure.in
+configure: configure.in config/acx.m4
 EOF
 }
 

[-- Attachment #3: Type: text/plain, Size: 289 bytes --]


-- 
Alexandre Oliva   Enjoy Guarana', see http://www.ic.unicamp.br/~oliva/
Red Hat GCC Developer                 aoliva@{redhat.com, gcc.gnu.org}
CS PhD student at IC-Unicamp        oliva@{lsd.ic.unicamp.br, gnu.org}
Free Software Evangelist                Professional serial bug killer

Re: (toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

[Daniel Jacobowitz]

On Sat, Dec 28, 2002 at 02:24:00PM -0200, Alexandre Oliva wrote:
> On Dec 28, 2002, Nathanael Nerode <neroden@twcny.rr.com> wrote:
> 
> > Unfortunately, I really have to go to sleep now. :-(
> 
> Fair enough.  Hope you had a good night :-)
> 
> > Please feel free to make this change on your own; it's obviously 
> > correct as far as I'm concerned.
> 
> Done.  While at that, I fixed a few other fallouts.  I'm checking in
> the patch below.

I notice that you moved the .NOTPARALLEL down to the configure section. 
While it is there what do you think of removing the serialization
dependencies?

Also, top level configure no longer accepts --norecursion.  This means
that one has to configure from scratch in any existing working
directory.  Is it as simple as allowing --norecursion, or is everything
expected to break anyway?


-- 
Daniel Jacobowitz
MontaVista Software                         Debian GNU/Linux Developer

Re: (toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

[Alexandre Oliva]

On Dec 28, 2002, Daniel Jacobowitz <drow@mvista.com> wrote:

> I notice that you moved the .NOTPARALLEL down to the configure section. 

Just because otherwise `make NOTPARALLEL=something' would cause make
to try to build something, not all.

> While it is there what do you think of removing the serialization
> dependencies?

Some day, some day...

Really, the only way to avoid them is to introduce some form of
locking, an idea that DJ vehemently rejects.

Unless...  We could perhaps have NOTPARALLEL set by default, which
would take care of avoiding configurations in parallel even without
serialized dependencies, but a configure option to disable NOTPARALLEL
and introduce locking.  DJ, how does this sound for you?

> Also, top level configure no longer accepts --norecursion.

The autoconf spelling is --no-recursion.  Anyway, it no longer
recurses...  Unfortunately, with autoconf, it gets *really* tricky to
introduce options that are not in the autoconf option space (--with or
--enable), so offering --norecursion for backward-compatibility gets
tricky.

> This means that one has to configure from scratch in any existing
> working directory.

./config.status --recheck?


Oh, perhaps you mean with a config.status that pre-dates
autoconfiscation?  Yeah, this would be tricky...  Editing
config.status is your best bet.  Failing that, well...  Running the
top-level configure again is no such big deal, is it? :-)

-- 
Alexandre Oliva   Enjoy Guarana', see http://www.ic.unicamp.br/~oliva/
Red Hat GCC Developer                 aoliva@{redhat.com, gcc.gnu.org}
CS PhD student at IC-Unicamp        oliva@{lsd.ic.unicamp.br, gnu.org}
Free Software Evangelist                Professional serial bug killer

Re: (toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

[Daniel Jacobowitz]

On Sat, Dec 28, 2002 at 03:51:19PM -0200, Alexandre Oliva wrote:
> On Dec 28, 2002, Daniel Jacobowitz <drow@mvista.com> wrote:
> 
> > I notice that you moved the .NOTPARALLEL down to the configure section. 
> 
> Just because otherwise `make NOTPARALLEL=something' would cause make
> to try to build something, not all.
> 
> > While it is there what do you think of removing the serialization
> > dependencies?
> 
> Some day, some day...
> 
> Really, the only way to avoid them is to introduce some form of
> locking, an idea that DJ vehemently rejects.
> 
> Unless...  We could perhaps have NOTPARALLEL set by default, which
> would take care of avoiding configurations in parallel even without
> serialized dependencies, but a configure option to disable NOTPARALLEL
> and introduce locking.  DJ, how does this sound for you?

It's a question of whether you can do the locking without making people
throw up, I think.

> > Also, top level configure no longer accepts --norecursion.
> 
> The autoconf spelling is --no-recursion.  Anyway, it no longer
> recurses...  Unfortunately, with autoconf, it gets *really* tricky to
> introduce options that are not in the autoconf option space (--with or
> --enable), so offering --norecursion for backward-compatibility gets
> tricky.
> 
> > This means that one has to configure from scratch in any existing
> > working directory.
> 
> ./config.status --recheck?
> 
> 
> Oh, perhaps you mean with a config.status that pre-dates
> autoconfiscation?  Yeah, this would be tricky...  Editing
> config.status is your best bet.  Failing that, well...  Running the
> top-level configure again is no such big deal, is it? :-)

You mean, like any config.status from any day before today? :)  Yes,
that's what I meant.  At least we get an error message.

-- 
Daniel Jacobowitz
MontaVista Software                         Debian GNU/Linux Developer

Re: (toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

[Alexandre Oliva]

On Dec 28, 2002, Daniel Jacobowitz <drow@mvista.com> wrote:

> It's a question of whether you can do the locking without making people
> throw up, I think.

I'm confident I can, but...  On second thought, it occurs to me that
all locking would accomplish is let one make in a make -j pool run one
configure script while other makes block on the lock.  Not good...

Another idea is that we could really run multiple configure scripts in
parallel, giving each one a separate cache file.  We'd still have to
synchronize the actions of taking a copy of config.cache and of
updating it, and then we might get some tests run more than once in
separate directories.


But then, frankly...  If running configure scripts in parallel is so
much of a problem, why haven't we had problems so far?  It's not like
our build infrastructure has ever prevented configure scripts in
sibling directories from running in parallel.  It's perfectly possible
for say gas, ld and binutils to be reconfigured in parallel after
their configure scripts are modified, and I've never heard of anyone
having problems because of this.

So, if we really can't avoid the problem, is it really worth fussing
so much about it?  I mean, if you want to make absolutely sure that
configure to be run sequentially, run `make configure' without -j and
be done with it, and only then run `make -jN all NOTPARALLEL=all'.  If
you can live with a little bit of risk, skip `make configure' and let
us know if you run into problems (after forced serialization of
configure is taken out, of course :-)  Comments?

> You mean, like any config.status from any day before today? :)  Yes,
> that's what I meant.

Sorry, it took me that long to figure out that's what you meant

-- 
Alexandre Oliva   Enjoy Guarana', see http://www.ic.unicamp.br/~oliva/
Red Hat GCC Developer                 aoliva@{redhat.com, gcc.gnu.org}
CS PhD student at IC-Unicamp        oliva@{lsd.ic.unicamp.br, gnu.org}
Free Software Evangelist                Professional serial bug killer

Re: (toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

[Dan Kegel]

Alexandre Oliva wrote:
> On Dec 28, 2002, Daniel Jacobowitz <drow@mvista.com> wrote:
> 
> 
>>It's a question of whether you can do the locking without making people
>>throw up, I think.
> 
> 
> I'm confident I can, but...  On second thought, it occurs to me that
> all locking would accomplish is let one make in a make -j pool run one
> configure script while other makes block on the lock.  Not good...

I dunno, sounds fine to me if it lets the subsequent real build
go in parallel.

-- 
Dan Kegel
Linux User #78045
http://www.kegel.com

Re: (toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

[DJ Delorie]

> Unless...  We could perhaps have NOTPARALLEL set by default, which
> would take care of avoiding configurations in parallel even without
> serialized dependencies, but a configure option to disable NOTPARALLEL
> and introduce locking.  DJ, how does this sound for you?

I am not "vehemently" opposed to it.  I've just seen, through
experience, that there are lots of ways for it to go wrong.
Serialized dependencies is a lot simpler and less error-prone (once we
get them right, nobody's commented on my last suggestion).  If we had
a simple locking mechanism that worked on all the
ext/nfs/dos/samba/etc/distcc filesystems we need to support, I'd be OK
with it.  I just think that people underestimate how much effort it
takes to do locking reliably.

The problem with NOTPARALLEL is that it shuts of *ALL* the toplevel
parallelisms, including building A while configuring B, and testing in
parallel.  The toplevel Makefile really needs to allow parallel jobs
to take advantage of the work we've done.  We just need to be able to
identify the parts that can't be done simultaneously, and somehow
prevent those.

Re: (toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

[Daniel Jacobowitz]

On Sat, Dec 28, 2002 at 04:34:38PM -0200, Alexandre Oliva wrote:
> On Dec 28, 2002, Daniel Jacobowitz <drow@mvista.com> wrote:
> 
> > It's a question of whether you can do the locking without making people
> > throw up, I think.
> 
> I'm confident I can, but...  On second thought, it occurs to me that
> all locking would accomplish is let one make in a make -j pool run one
> configure script while other makes block on the lock.  Not good...
> 
> Another idea is that we could really run multiple configure scripts in
> parallel, giving each one a separate cache file.  We'd still have to
> synchronize the actions of taking a copy of config.cache and of
> updating it, and then we might get some tests run more than once in
> separate directories.
> 
> 
> But then, frankly...  If running configure scripts in parallel is so
> much of a problem, why haven't we had problems so far?  It's not like
> our build infrastructure has ever prevented configure scripts in
> sibling directories from running in parallel.  It's perfectly possible
> for say gas, ld and binutils to be reconfigured in parallel after
> their configure scripts are modified, and I've never heard of anyone
> having problems because of this.
> 
> So, if we really can't avoid the problem, is it really worth fussing
> so much about it?  I mean, if you want to make absolutely sure that
> configure to be run sequentially, run `make configure' without -j and
> be done with it, and only then run `make -jN all NOTPARALLEL=all'.  If
> you can live with a little bit of risk, skip `make configure' and let
> us know if you run into problems (after forced serialization of
> configure is taken out, of course :-)  Comments?

Now you're speaking my language.  This even lets me say "make
configure-gdb" to configure exactly the directories I need.  Sure, some
reconfigures may end up taking place in parallel; I posit that if we
see a config.cache problem then it should be solved by fixing
_autoconf_ to update the cache atomically.

-- 
Daniel Jacobowitz
MontaVista Software                         Debian GNU/Linux Developer

Re: (toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

[Alexandre Oliva]

On Dec 28, 2002, Alexandre Oliva <aoliva@redhat.com> wrote:

> But then, frankly...  If running configure scripts in parallel is so
> much of a problem, why haven't we had problems so far?  It's not like
> our build infrastructure has ever prevented configure scripts in
> sibling directories from running in parallel.

In fact, target libraries have had their configure scripts run in
parallel for quite a while (since 2000-07-24).  Has anyone ever run
into a problem because of this?

-- 
Alexandre Oliva   Enjoy Guarana', see http://www.ic.unicamp.br/~oliva/
Red Hat GCC Developer                 aoliva@{redhat.com, gcc.gnu.org}
CS PhD student at IC-Unicamp        oliva@{lsd.ic.unicamp.br, gnu.org}
Free Software Evangelist                Professional serial bug killer

Re: (toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

[DJ Delorie]

> Unfortunately, I really have to go to sleep now.
> 
> Part of the motivation for autoconfiscating was to make the top level less 
> of a "secret region" which only the "initiated" could hack on.  I hope 
> I've accomplished that.

In case nobody else says so explicitly, please accept our thanks for
all the effort you've put into this task.

Re: (toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

[DJ Delorie]

> Now you're speaking my language.  This even lets me say "make
> configure-gdb" to configure exactly the directories I need.  Sure, some
> reconfigures may end up taking place in parallel; I posit that if we
> see a config.cache problem then it should be solved by fixing
> _autoconf_ to update the cache atomically.

Not just update, but *merge*.  Autoconf reads the cache, does lots of
processing, then writes the cache.  The cache may have been updated
during the processing, and you don't want to toss those updates.

But it would be nice if autoconf managed this problem for us.  It
would mean migrating to a new version of autoconf, of course.

Re: (toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

[DJ Delorie]

> I dunno, sounds fine to me if it lets the subsequent real build
> go in parallel.

If you fill the job slots with busy-wait configures, there aren't any
slots left to put real builds in.

Of course, this is the point where we could conditionally take
advantage of gmake's "don't run these in parallel" syntax.

Re: (toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

[Daniel Jacobowitz]

On Sat, Dec 28, 2002 at 02:00:58PM -0500, DJ Delorie wrote:
> 
> > I dunno, sounds fine to me if it lets the subsequent real build
> > go in parallel.
> 
> If you fill the job slots with busy-wait configures, there aren't any
> slots left to put real builds in.
> 
> Of course, this is the point where we could conditionally take
> advantage of gmake's "don't run these in parallel" syntax.

Which it doesn't have.  I was mistaken about the purpose of the
construct I quoted.  Not that we couldn't add such a syntax to GNU
make; it might be generally useful.

-- 
Daniel Jacobowitz
MontaVista Software                         Debian GNU/Linux Developer

Re: (toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

[Alexandre Oliva]

On Dec 28, 2002, DJ Delorie <dj@delorie.com> wrote:

> Not just update, but *merge*.  Autoconf reads the cache, does lots of
> processing, then writes the cache.  The cache may have been updated
> during the processing, and you don't want to toss those updates.

Why not?  It's just a cache, after all!  Worst case, when you
reconfigure, you get to rerun some tests.  So what?

Also, I've been measuring the size of config.cache in typical
configurations, and I find it to always be about 20Kb for a full
uberbaum build.  The way autoconf updates it is to create a file named
confcache, with the new contents of the cache, and then runs `cat
confcache > $cache_file'.  I believe it is safe to assume that writes
that are this small are atomic on any reasonable OS, and the window
for two or more simultaneous configure scripts to truncate the file at
the same time, and then get a larger write() to complete faster than a
smaller one, such that the smaller write doesn't completely overwrite
the contents of the larger one, is so small that it's negligible.

Just consider that updating config.cache is something that happens,
what, at most 200 times in a full build, with a failure window that
lasts a few microseconds in a process that can take about an hour.

I'm not disputing that it can possibly fail.  I just think the odds
are so low that it's not worth worrying about.  I'd at least like to
be able to control this behavior with configure options.  I'm working
on something that will require configure options to disable the
serialized configurations, but more and more I think we might as well
turn them on by default.  It's not like make -j is default or
anything, and whoever uses make -j might also be expected to configure
with an option that makes the build 100% -j-safe.

> But it would be nice if autoconf managed this problem for us.

It can't.  Consider that the config.cache file specified in the
command line may be in a directory that is not writable, and that two
totally unrelated builds may be using the same config.cache file.  How
could autoconf possibly synchronize them so as to not get in trouble?
To add to the fun, assume that the two configures are running in
separate machines, whose only shared filesystem is the one containing
config.cache, and such that there's no write permission on it anywhere
except in the config.cache file.

Of course, it *might* try to create a lock file in the directory
containing config.cache (but then it might be a soft link elsewhere),
but do you really want to be creating files all over the place that
might be left dangling should the machine reboot?  And then, consider
that configure has really no way to tell something like
--cache-file=/some/random/shared/config.cache from what it gets from
the top-level Makefile, namely --cache-file=../../../config.cache
(which might have been turned into a full, yet possibly non-canonical
pathname such as --cache-file=`pwd`/config.cache).


That said, using some autoconf 2.5x features, we may be able to come
up with a configure-like script whose only purpose is to merge two
config.cache files together, such that we could then try to be smarter
and give each sub-package its own config.cache, just primed with the
global cache and having the global cache updated (mostly)atomically
afterwards.  I say mostly because, if there are other, unrelated
builds using the same global cache file, there's really nothing we can
do to prevent them from conflicting.

-- 
Alexandre Oliva   Enjoy Guarana', see http://www.ic.unicamp.br/~oliva/
Red Hat GCC Developer                 aoliva@{redhat.com, gcc.gnu.org}
CS PhD student at IC-Unicamp        oliva@{lsd.ic.unicamp.br, gnu.org}
Free Software Evangelist                Professional serial bug killer

Re: (toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

[Alexandre Oliva]

[-- Attachment #1: Type: text/plain, Size: 3490 bytes --]

On Dec 28, 2002, Alexandre Oliva <aoliva@redhat.com> wrote:

> I'm working on something that will require configure options to
> disable the serialized configurations, but more and more I think we
> might as well turn them on by default.  It's not like make -j is
> default or anything, and whoever uses make -j might also be expected
> to configure with an option that makes the build 100% -j-safe.

Here's the patch I came up with.  I'm not putting it in for now (which
is why there are no ChangeLogs), and I'm not documenting the new
configure options anywhere, since they're still open for debate.  It
also fixes a number of problems:

- for some reason I don't understand, the multilib.out/multilib.ts
  pair was causing maybe-all-gcc to be resolved twice.  It appears
  that make was being re-entered to rebuild multilib.ts even though it
  wasn't actually necessary.  I came up with a simpler construct that
  amounts to the same result.  After all, multilib.ts was always going
  to be newer than multilib.out anyway, and we always constructed
  multilib.ts anyway, so...

- using the Makefiles to represent the actual configure-* targets was
  rendering the explicit dependencies of configure-* targets in the
  `Dependencies between different modules' useless.  Since say
  configure-foo depended on maybe-configure-bar and foo/Makefile, they
  could be built in parallel, and then configure-foo would be built
  with a do-nothing.  I've rearranged for us to go back to the phony
  configure-* targets, and added explicit tests for the target's
  Makefile to short-circuit the configure process.  To make sure
  target libraries are reconfigured when multilib.out changes, the
  update of multilib.out removes the target library's Makefile from
  the build dir, forcing its configure to run.

- I've also removed the dependence of configure-* on ./config.status.
  I don't understand why we put this in.  I get the impression it was
  a mistake, since the earlier top-level (say in the gcc-3.3 branch)
  does not get any sub-packages reconfigured just because the top
  level config.status changes.  If we really want to do this, we
  should arrange for the code that updates config.status in the
  top-level to also remove the Makefiles of all subdirs, so that
  they're reconfigured next.

- I've made sure that maybedep.tmp and serdep.tmp are never empty, and
  that they contain a marker indicating where these pieces of Makefile
  content came from.

- the changes to introduce the new configure options,
  --disable-serial-configure, --disable-serial-build-configure,
  --disable-serial-host-configure and --disable-serial-target-configure,
  are the few `test' lines added to configure.in.  Should they be
  accepted, I'd add calls to AC_ARG_ENABLE to get them documented in
  configure --help, and to the top-level configure docs.  I'd much
  rather have all serial configure disabled by default, though.

I'm also thinking we might be able to diagnose the presence of
-j in MAKEFLAGS and warn in case the serial dependencies are not
enabled, or even arrange for the Makefile to be automatically
regenerated containing such dependencies.  Something like this in the
default serdep.tmp:

SERIALIZE_PARALLEL_CONFIGURE = force
serdep.tmp: $(SERIALIZE_PARALLEL_CONFIGURE)
        @if echo $(MAKEFLAGS) | grep -e -j > /dev/null; then \
          echo Warning: parallel make used serial dependencies >&2; \
          cp serdep.par serdep.tmp; \
        fi
Makefile: serdep.tmp


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: config-parallel.patch --]
[-- Type: text/x-patch, Size: 6125 bytes --]

Index: Makefile.tpl
===================================================================
RCS file: /cvs/gcc/gcc/Makefile.tpl,v
retrieving revision 1.26
diff -u -p -r1.26 Makefile.tpl
--- Makefile.tpl 28 Dec 2002 16:24:47 -0000 1.26
+++ Makefile.tpl 28 Dec 2002 21:00:12 -0000
@@ -641,7 +641,7 @@ local-clean:
 
 local-distclean:
 	-rm -f Makefile config.status config.cache mh-frag mt-frag
-	-rm -f multilib.out multilib.ts multilib.tmp maybedep.tmp serdep.tmp
+	-rm -f multilib.out multilib.tmp maybedep.tmp serdep.tmp
 	-if [ "$(TARGET_SUBDIR)" != "." ]; then \
 	  rm -rf $(TARGET_SUBDIR); \
 	else true; fi
@@ -789,9 +789,10 @@ TAGS: do-TAGS
 [+ FOR build_modules +]
 .PHONY: configure-build-[+module+] maybe-configure-build-[+module+]
 maybe-configure-build-[+module+]:
-configure-build-[+module+]: $(BUILD_SUBDIR)/[+module+]/Makefile
-$(BUILD_SUBDIR)/[+module+]/Makefile: config.status
-	@[ -d $(BUILD_SUBDIR)/[+module+] ] || mkdir $(BUILD_SUBDIR)/[+module+];\
+configure-build-[+module+]:
+	@test ! -f $(BUILD_SUBDIR)/[+module+]/Makefile || exit 0; \
+	    [ -d $(BUILD_SUBDIR)/[+module+] ] || \
+		mkdir $(BUILD_SUBDIR)/[+module+];\
 	    r=`${PWD}`; export r; \
 	    s=`cd $(srcdir); ${PWD}`; export s; \
 	    AR="$(AR_FOR_BUILD)"; export AR; \
@@ -862,10 +863,9 @@ all-build-[+module+]: configure-build-[+
 [+ FOR host_modules +]
 .PHONY: configure-[+module+] maybe-configure-[+module+]
 maybe-configure-[+module+]:
-configure-[+module+]: [+module+]/Makefile
-
-[+module+]/Makefile: config.status
-	@[ -d [+module+] ] || mkdir [+module+]; \
+configure-[+module+]:
+	@test ! -f [+module+]/Makefile || exit 0; \
+	[ -d [+module+] ] || mkdir [+module+]; \
 	r=`${PWD}`; export r; \
 	s=`cd $(srcdir); ${PWD}`; export s; \
 	CC="$(CC)"; export CC; \
@@ -964,15 +964,18 @@ install-[+module+]: installdirs
 [+ FOR target_modules +]
 .PHONY: configure-target-[+module+] maybe-configure-target-[+module+]
 maybe-configure-target-[+module+]:
-configure-target-[+module+]: $(TARGET_SUBDIR)/[+module+]/Makefile
 
 # There's only one multilib.out.  Cleverer subdirs shouldn't need it copied.
 $(TARGET_SUBDIR)/[+module+]/multilib.out: multilib.out
-	@[ -d $(TARGET_SUBDIR)/[+module+] ] || mkdir $(TARGET_SUBDIR)/[+module+];\
+	@[ -d $(TARGET_SUBDIR)/[+module+] ] || \
+	    mkdir $(TARGET_SUBDIR)/[+module+]; \
+	rm -f $(TARGET_SUBDIR)/[+module+]/Makefile || : ; \
 	cp multilib.out $(TARGET_SUBDIR)/[+module+]/multilib.out
 
-$(TARGET_SUBDIR)/[+module+]/Makefile: config.status $(TARGET_SUBDIR)/[+module+]/multilib.out
-	@[ -d $(TARGET_SUBDIR)/[+module+] ] || mkdir $(TARGET_SUBDIR)/[+module+];\
+configure-target-[+module+]: $(TARGET_SUBDIR)/[+module+]/multilib.out
+	@test ! -f $(TARGET_SUBDIR)/[+module+]/Makefile || exit 0; \
+	    [ -d $(TARGET_SUBDIR)/[+module+] ] || \
+		mkdir $(TARGET_SUBDIR)/[+module+];\
 	    r=`${PWD}`; export r; \
 	    s=`cd $(srcdir); ${PWD}`; export s; \
 	    $(SET_LIB_PATH) \
@@ -1092,10 +1095,9 @@ install-target-[+module+]: installdirs
 # gcc is the only module which uses GCC_FLAGS_TO_PASS.
 .PHONY: configure-gcc maybe-configure-gcc
 maybe-configure-gcc:
-configure-gcc: gcc/Makefile
-
-gcc/Makefile: config.status
-	@[ -d gcc ] || mkdir gcc; \
+configure-gcc:
+	@test ! -f gcc/Makefile || exit 0; \
+	[ -d gcc ] || mkdir gcc; \
 	r=`${PWD}`; export r; \
 	s=`cd $(srcdir); ${PWD}`; export s; \
 	CC="$(CC)"; export CC; \
@@ -1395,17 +1397,11 @@ configure-target-qthreads: $(ALL_GCC_C)
 # work around various timestamp bugs on some systems.
 # We use move-if-change so that it's only considered updated when it
 # actually changes, because it has to depend on a phony target.
-multilib.out: multilib.ts
-	@if [ -f multilib.out ] ; then : else \
-	  rm -f multilib.ts; $(MAKE) multilib.ts; \
-	fi
-
-multilib.ts: maybe-all-gcc
+multilib.out: maybe-all-gcc
 	@r=`${PWD}`; export r; \
 	echo "Checking multilib configuration..."; \
 	$(CC_FOR_TARGET) --print-multi-lib > multilib.tmp 2> /dev/null ; \
 	$(SHELL) $(srcdir)/move-if-change multilib.tmp multilib.out ; \
-	echo timestamp > multilib.ts
 
 # Rebuilding Makefile.in, using autogen.
 AUTOGEN = autogen
Index: configure.in
===================================================================
RCS file: /cvs/gcc/gcc/configure.in,v
retrieving revision 1.202
diff -u -p -r1.202 configure.in
--- configure.in 28 Dec 2002 17:57:14 -0000 1.202
+++ configure.in 28 Dec 2002 21:00:13 -0000
@@ -1686,6 +1686,7 @@ esac
 
 # Create the 'maybe dependencies'.  This uses a temporary file.
 rm -f maybedep.tmp
+echo '# maybedep.tmp' > maybedep.tmp
 for item in ${all_build_modules} ${all_host_modules} ${all_target_modules} \
 	${install_host_modules} ${install_target_modules} \
 	${configure_build_modules} ${configure_host_modules} ${configure_target_modules} \
@@ -1700,27 +1701,34 @@ AC_SUBST_FILE(maybe_dependencies)
 # These force 'configure's to be done one at a time, to avoid problems
 # with contention over a shared config.cache.
 rm -f serdep.tmp
+echo '# serdep.tmp' > serdep.tmp
 olditem=
+test "x${enable_serial_configure}" = xno ||
+test "x${enable_serial_build_configure}" = xno ||
 for item in ${build_configdirs} ; do
   case ${olditem} in
     "") ;;
-    *) echo "\$(BUILD_SUBDIR)/${item}/Makefile: \$(BUILD_SUBDIR)/${olditem}/Makefile" >> serdep.tmp ;;
+    *) echo "configure-build-${item}: configure-build-${olditem}" >> serdep.tmp ;;
   esac
   olditem=${item}
 done
 olditem=
+test "x${enable_serial_configure}" = xno ||
+test "x${enable_serial_host_configure}" = xno ||
 for item in ${configdirs} ; do
   case ${olditem} in
     "") ;;
-    *) echo "${item}/Makefile: ${olditem}/Makefile" >> serdep.tmp ;;
+    *) echo "configure-${item}: configure-${olditem}" >> serdep.tmp ;;
   esac
   olditem=${item}
 done
 olditem=
+test "x${enable_serial_configure}" = xno ||
+test "x${enable_serial_target_configure}" = xno ||
 for item in ${target_configdirs} ; do
   case ${olditem} in
     "") ;;
-    *) echo "\$(TARGET_SUBDIR)/${item}/Makefile: \$(TARGET_SUBDIR)/${olditem}/Makefile" >> serdep.tmp ;;
+    *) echo "configure-target-${item}: configure-target-${olditem}" >> serdep.tmp ;;
   esac
   olditem=${item}
 done

[-- Attachment #3: Type: text/plain, Size: 289 bytes --]


-- 
Alexandre Oliva   Enjoy Guarana', see http://www.ic.unicamp.br/~oliva/
Red Hat GCC Developer                 aoliva@{redhat.com, gcc.gnu.org}
CS PhD student at IC-Unicamp        oliva@{lsd.ic.unicamp.br, gnu.org}
Free Software Evangelist                Professional serial bug killer

Re: (toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

[DJ Delorie]

> Why not?  It's just a cache, after all!  Worst case, when you
> reconfigure, you get to rerun some tests.  So what?

Worst case is config.cache gets corrupt values, and the build breaks.
I agree it's rare.  If we understand the risk, and collectively decide
to live with it until an actual problem happens, I'm OK with that.

> > But it would be nice if autoconf managed this problem for us.
> 
> It can't.  Consider that the config.cache file specified in the
> command line may be in a directory that is not writable,

That's a different type of problem.  Does autoconf allow for failure
to write to config.cache?

Re: (toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

[Alexandre Oliva]

On Dec 28, 2002, DJ Delorie <dj@delorie.com> wrote:

>> > But it would be nice if autoconf managed this problem for us.
>> 
>> It can't.  Consider that the config.cache file specified in the
>> command line may be in a directory that is not writable,

> That's a different type of problem.  Does autoconf allow for failure
> to write to config.cache?

It checks whether the file is writable, and then it overwrites the
file.  Whether the directory is writable doesn't matter.  It just
makes it harder to choose the location of the lock file.

-- 
Alexandre Oliva   Enjoy Guarana', see http://www.ic.unicamp.br/~oliva/
Red Hat GCC Developer                 aoliva@{redhat.com, gcc.gnu.org}
CS PhD student at IC-Unicamp        oliva@{lsd.ic.unicamp.br, gnu.org}
Free Software Evangelist                Professional serial bug killer

Re: (toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

[Alexandre Oliva]

[-- Attachment #1: Type: text/plain, Size: 621 bytes --]

In addition to the patch I posted yesterday, this one introduces
documentation for --disable-serial-configure and freinds in the output
of configure --help, and removes the nonopt argument from baseargs, so
that we don't pass --build=foo --host=foo foo down when the user runs
configure foo.  Christian Jönsson had run into this bug; this should
fix it.

Other than the fixes, this patch should not introduce any real changes
in behavior.  serial configure remains enabled by default.  I've
tested it with and without the options, and the generated serdep.tmp
files was as expected in all cases.

I'm checking this in.


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: toplev-fixes.patch --]
[-- Type: text/x-patch, Size: 8783 bytes --]

Index: ChangeLog
from  Alexandre Oliva  <aoliva@redhat.com>

	* Makefile.tpl (local-distclean): Don't remove...
	(multilib.ts): ... this.  Moved into...
	(multilib.out): ... this.  Don't use sub-make.
	($(BUILD_SUBDIR)/[+module+]/Makefile, [+module+]/Makefile,
	$(TARGET_SUBDIR)/[+module+]/Makefile, gcc/Makefile): Moved into...
	(configure-build-[+module+], configure-[+module+],
	configure-target-[+module+], configure-gcc): ... these.  Test
	for Makefile existence.  Drop config.status from dependencies.
	* Makefile.in: Rebuilt.
	* configure.in: Move gcc-version-trigger to the end of
	ac_configure_args.  Add comments to maybedep.tmp and
	serdep.tmp.  Introduce --disable-serial-configure.  Remove
	nonopt from baseargs, matching and removing corresponding
	whitespace while at it.
	* configure: Rebuilt.

Index: Makefile.tpl
===================================================================
RCS file: /cvs/gcc/gcc/Makefile.tpl,v
retrieving revision 1.26
diff -u -p -r1.26 Makefile.tpl
--- Makefile.tpl 28 Dec 2002 16:24:47 -0000 1.26
+++ Makefile.tpl 29 Dec 2002 18:47:33 -0000
@@ -641,7 +641,7 @@ local-clean:
 
 local-distclean:
 	-rm -f Makefile config.status config.cache mh-frag mt-frag
-	-rm -f multilib.out multilib.ts multilib.tmp maybedep.tmp serdep.tmp
+	-rm -f multilib.out multilib.tmp maybedep.tmp serdep.tmp
 	-if [ "$(TARGET_SUBDIR)" != "." ]; then \
 	  rm -rf $(TARGET_SUBDIR); \
 	else true; fi
@@ -789,9 +789,10 @@ TAGS: do-TAGS
 [+ FOR build_modules +]
 .PHONY: configure-build-[+module+] maybe-configure-build-[+module+]
 maybe-configure-build-[+module+]:
-configure-build-[+module+]: $(BUILD_SUBDIR)/[+module+]/Makefile
-$(BUILD_SUBDIR)/[+module+]/Makefile: config.status
-	@[ -d $(BUILD_SUBDIR)/[+module+] ] || mkdir $(BUILD_SUBDIR)/[+module+];\
+configure-build-[+module+]:
+	@test ! -f $(BUILD_SUBDIR)/[+module+]/Makefile || exit 0; \
+	    [ -d $(BUILD_SUBDIR)/[+module+] ] || \
+		mkdir $(BUILD_SUBDIR)/[+module+];\
 	    r=`${PWD}`; export r; \
 	    s=`cd $(srcdir); ${PWD}`; export s; \
 	    AR="$(AR_FOR_BUILD)"; export AR; \
@@ -862,10 +863,9 @@ all-build-[+module+]: configure-build-[+
 [+ FOR host_modules +]
 .PHONY: configure-[+module+] maybe-configure-[+module+]
 maybe-configure-[+module+]:
-configure-[+module+]: [+module+]/Makefile
-
-[+module+]/Makefile: config.status
-	@[ -d [+module+] ] || mkdir [+module+]; \
+configure-[+module+]:
+	@test ! -f [+module+]/Makefile || exit 0; \
+	[ -d [+module+] ] || mkdir [+module+]; \
 	r=`${PWD}`; export r; \
 	s=`cd $(srcdir); ${PWD}`; export s; \
 	CC="$(CC)"; export CC; \
@@ -964,15 +964,18 @@ install-[+module+]: installdirs
 [+ FOR target_modules +]
 .PHONY: configure-target-[+module+] maybe-configure-target-[+module+]
 maybe-configure-target-[+module+]:
-configure-target-[+module+]: $(TARGET_SUBDIR)/[+module+]/Makefile
 
 # There's only one multilib.out.  Cleverer subdirs shouldn't need it copied.
 $(TARGET_SUBDIR)/[+module+]/multilib.out: multilib.out
-	@[ -d $(TARGET_SUBDIR)/[+module+] ] || mkdir $(TARGET_SUBDIR)/[+module+];\
+	@[ -d $(TARGET_SUBDIR)/[+module+] ] || \
+	    mkdir $(TARGET_SUBDIR)/[+module+]; \
+	rm -f $(TARGET_SUBDIR)/[+module+]/Makefile || : ; \
 	cp multilib.out $(TARGET_SUBDIR)/[+module+]/multilib.out
 
-$(TARGET_SUBDIR)/[+module+]/Makefile: config.status $(TARGET_SUBDIR)/[+module+]/multilib.out
-	@[ -d $(TARGET_SUBDIR)/[+module+] ] || mkdir $(TARGET_SUBDIR)/[+module+];\
+configure-target-[+module+]: $(TARGET_SUBDIR)/[+module+]/multilib.out
+	@test ! -f $(TARGET_SUBDIR)/[+module+]/Makefile || exit 0; \
+	    [ -d $(TARGET_SUBDIR)/[+module+] ] || \
+		mkdir $(TARGET_SUBDIR)/[+module+];\
 	    r=`${PWD}`; export r; \
 	    s=`cd $(srcdir); ${PWD}`; export s; \
 	    $(SET_LIB_PATH) \
@@ -1092,10 +1095,9 @@ install-target-[+module+]: installdirs
 # gcc is the only module which uses GCC_FLAGS_TO_PASS.
 .PHONY: configure-gcc maybe-configure-gcc
 maybe-configure-gcc:
-configure-gcc: gcc/Makefile
-
-gcc/Makefile: config.status
-	@[ -d gcc ] || mkdir gcc; \
+configure-gcc:
+	@test ! -f gcc/Makefile || exit 0; \
+	[ -d gcc ] || mkdir gcc; \
 	r=`${PWD}`; export r; \
 	s=`cd $(srcdir); ${PWD}`; export s; \
 	CC="$(CC)"; export CC; \
@@ -1395,17 +1397,11 @@ configure-target-qthreads: $(ALL_GCC_C)
 # work around various timestamp bugs on some systems.
 # We use move-if-change so that it's only considered updated when it
 # actually changes, because it has to depend on a phony target.
-multilib.out: multilib.ts
-	@if [ -f multilib.out ] ; then : else \
-	  rm -f multilib.ts; $(MAKE) multilib.ts; \
-	fi
-
-multilib.ts: maybe-all-gcc
+multilib.out: maybe-all-gcc
 	@r=`${PWD}`; export r; \
 	echo "Checking multilib configuration..."; \
 	$(CC_FOR_TARGET) --print-multi-lib > multilib.tmp 2> /dev/null ; \
 	$(SHELL) $(srcdir)/move-if-change multilib.tmp multilib.out ; \
-	echo timestamp > multilib.ts
 
 # Rebuilding Makefile.in, using autogen.
 AUTOGEN = autogen
Index: configure.in
===================================================================
RCS file: /cvs/gcc/gcc/configure.in,v
retrieving revision 1.202
diff -u -p -r1.202 configure.in
--- configure.in 28 Dec 2002 17:57:14 -0000 1.202
+++ configure.in 29 Dec 2002 18:47:37 -0000
@@ -112,7 +112,7 @@ else
         ;;
       * )
         # Add to all subconfigure arguments: build, host, and target.
-        ac_configure_args="--with-gcc-version-trigger=$gcc_version_trigger $ac_configure_args"
+        ac_configure_args="$ac_configure_args --with-gcc-version-trigger=$gcc_version_trigger"
         ;;
     esac
   fi
@@ -1686,6 +1686,7 @@ esac
 
 # Create the 'maybe dependencies'.  This uses a temporary file.
 rm -f maybedep.tmp
+echo '# maybedep.tmp' > maybedep.tmp
 for item in ${all_build_modules} ${all_host_modules} ${all_target_modules} \
 	${install_host_modules} ${install_target_modules} \
 	${configure_build_modules} ${configure_host_modules} ${configure_target_modules} \
@@ -1697,45 +1698,61 @@ AC_SUBST_FILE(maybe_dependencies)
 
 # Create the serialization dependencies.  This uses a temporary file.
 
+AC_ARG_ENABLE([serial-configure],
+[  --disable-serial-[{host,target,build}-]configure
+                          Don't force sequential configuration of
+                          sub-packages for the host, target or build
+			  machine, or of any sub-packages at all])
+
 # These force 'configure's to be done one at a time, to avoid problems
 # with contention over a shared config.cache.
 rm -f serdep.tmp
+echo '# serdep.tmp' > serdep.tmp
 olditem=
+test "x${enable_serial_configure}" = xno ||
+test "x${enable_serial_build_configure}" = xno ||
 for item in ${build_configdirs} ; do
   case ${olditem} in
     "") ;;
-    *) echo "\$(BUILD_SUBDIR)/${item}/Makefile: \$(BUILD_SUBDIR)/${olditem}/Makefile" >> serdep.tmp ;;
+    *) echo "configure-build-${item}: configure-build-${olditem}" >> serdep.tmp ;;
   esac
   olditem=${item}
 done
 olditem=
+test "x${enable_serial_configure}" = xno ||
+test "x${enable_serial_host_configure}" = xno ||
 for item in ${configdirs} ; do
   case ${olditem} in
     "") ;;
-    *) echo "${item}/Makefile: ${olditem}/Makefile" >> serdep.tmp ;;
+    *) echo "configure-${item}: configure-${olditem}" >> serdep.tmp ;;
   esac
   olditem=${item}
 done
 olditem=
+test "x${enable_serial_configure}" = xno ||
+test "x${enable_serial_target_configure}" = xno ||
 for item in ${target_configdirs} ; do
   case ${olditem} in
     "") ;;
-    *) echo "\$(TARGET_SUBDIR)/${item}/Makefile: \$(TARGET_SUBDIR)/${olditem}/Makefile" >> serdep.tmp ;;
+    *) echo "configure-target-${item}: configure-target-${olditem}" >> serdep.tmp ;;
   esac
   olditem=${item}
 done
 serialization_dependencies=serdep.tmp
 AC_SUBST_FILE(serialization_dependencies)
 
-# Base args.  Strip norecursion, cache-file, srcdir, host, build, target.
-# These are the ones we might not want to pass down to subconfigures.
-baseargs=`echo "${ac_configure_args}" | \
-	sed -e 's/--no[[^ 	]]*//' \
-	    -e 's/--cache[[a-z-]]*=[[^ 	]]*//' \
-	    -e 's/--sr[[a-z-]]*=[[^ 	]]*//' \
-	    -e 's/--ho[[a-z-]]*=[[^ 	]]*//' \
-	    -e 's/--bu[[a-z-]]*=[[^ 	]]*//' \
-	    -e 's/--ta[[a-z-]]*=[[^ 	]]*//'`
+# Base args.  Strip norecursion, cache-file, srcdir, host, build,
+# target and nonopt.  These are the ones we might not want to pass
+# down to subconfigures.
+baseargs=`echo " ${ac_configure_args} " | \
+	sed -e 's/ --no[[^ ]]* / /' \
+	    -e 's/ --cache[[a-z-]]*=[[^ ]]* / /' \
+	    -e 's/ --sr[[a-z-]]*=[[^ ]]* / /' \
+	    -e 's/ --ho[[a-z-]]*=[[^ ]]* / /' \
+	    -e 's/ --bu[[a-z-]]*=[[^ ]]* / /' \
+	    -e 's/ --ta[[a-z-]]*=[[^ ]]* / /' \
+	    -e 's/ [[^-][^ ]*] / /' \
+	    -e 's/^ *//;s/ *$//'`
 
 # For the build-side libraries, we just need to pretend we're native,
 # and not use the same cache file.  Multilibs are neither needed nor

[-- Attachment #3: Type: text/plain, Size: 289 bytes --]


-- 
Alexandre Oliva   Enjoy Guarana', see http://www.ic.unicamp.br/~oliva/
Red Hat GCC Developer                 aoliva@{redhat.com, gcc.gnu.org}
CS PhD student at IC-Unicamp        oliva@{lsd.ic.unicamp.br, gnu.org}
Free Software Evangelist                Professional serial bug killer

Re: (toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

[Michael Elizabeth Chastain]

Ben Elliston writes:
> If, instead, the cache were a "dot" directory that contained files
> whose filenames had the form "key=value", then test results could be
> examined or updated using atomic file system operations and there
> should be no races.

The Linux kernel does something similar for its configuration variables
(which are different than autoconf cache entries): one file per variable.
The kernel does this to achieve fine-grained dependencies on the
configuration, so that changing CONFIG_DRIVER_FOO causes recompilation
of only the files that depend on CONFIG_DRIVER_FOO.

With a little care I think this would be thread safe.  One tricky point
is that you have to be prepared for a file to not exist at the moment
you open it because some other thread is doing an "unlink / rename" at
that moment.  Since this is a cache, that will simply cause some extra
cache misses.

There is a potential for a thundering-herd performance problem where every
thread wants to compute HAVE_FOO_H, they all deposit identical copies into
the cache, and then they move on and see that HAVE_BAR_H is not in the
cache either, lockstepping their way through similar configure scripts.
It would actually improve efficiency for autoconf to randomize the order
of tests in the configure script.  :)

Also, I think that you could mix the two cache styles in a single tree.
All the old-style subdirectories would use config.cache, and all
the new-style subdirectories would use the new fine-grained cache.
Serialization would be needed only for old-style "config.cache"
subdirectories.  New-style subdirectories could somehow say to autogen
"I am new-style, I use the new fine-grained cache, don't serialize my
configure goal with other configure goals."  There would be a performance
penalty for a mixed tree, which would go away when the last old-style
subdirectory got upgraded.

Michael C
back-seat driver :)

Re: (toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

[Ben Elliston]

>>>>> "Michael" == Michael Elizabeth Chastain <mec@shout.net> writes:

  Michael> I would also prefer no locks, because I don't want to deal
  Michael> with locking mechanism headaches when a user with, say, an
  Michael> SMP Irix NFS client and an HP/UX NFS server has funny build
  Michael> problems.

I have been pondering this problem.  It's a shame Autoconf just can't
be fixed (and patches even backported to 2.13, if necessary) to change
the structure of the cache.  These locking problems exist because the
cache values are kept in a single file and to control access via a
single lock would be too coarse grained.

If, instead, the cache were a "dot" directory that contained files
whose filenames had the form "key=value", then test results could be
examined or updated using atomic file system operations and there
should be no races.

  Michael> So I'm in favor of a dumb, simple, and reliable mechanism
  Michael> to serialize the configures.

I think the performance of large configure runs is becoming too poor
to take this line for much longer.

Ben

Re: (toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

[Alexandre Oliva]

On Dec 29, 2002, Michael Elizabeth Chastain <mec@shout.net> wrote:

> Perhaps you think of "make -j" as an inherently unsafe option.

Nope.  It's just not the default.  If you're using a non-default
option, why can't we ask you to also use another non-default option to
make it absolutely safe?

> I build gcc about 50 times per week with automated scripts.

Then you could tweak your automated scripts to enable the
safest-possible set up, in case you actually find it to be a problem.
I'm yet to hear of any actual occurrences of a problem.  I very much
doubt we're ever going to, even if we disable serialized dependencies.

-- 
Alexandre Oliva   Enjoy Guarana', see http://www.ic.unicamp.br/~oliva/
Red Hat GCC Developer                 aoliva@{redhat.com, gcc.gnu.org}
CS PhD student at IC-Unicamp        oliva@{lsd.ic.unicamp.br, gnu.org}
Free Software Evangelist                Professional serial bug killer

Re: (toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

[Michael Elizabeth Chastain]

Alexandre Oliva writes:
> Should we penalize everybody just because some might want to make -j
> and make it absolutely sure that they won't have any risk whatsoever
> of cache corruption and they won't miss any opportunities for caching
> test results.

In my humble opinion, yes.  If you have a choice of options,
make the default option be safe, and let people who want to go faster
with risk of corruption specify additional options.

Perhaps you think of "make -j" as an inherently unsafe option.
In 1999 I would have agreed.  In 2002 I think it's debatable.
As time passes, I think it will become more and more important
that it work without races.

I build gcc about 50 times per week with automated scripts.  Right now
I use a single-processor machine.  If I buy a multi-processor machine,
I would like to just turn on "make -j" and have it work.  When it doesn't
work, I want all the bugs that I see to be real bugs that I can report
to gcc-bugs and not have to check for cache corruption on every fail and
not have people closing my bug reports with "might be cache corruption,
don't use -j".

I know other people run automated and semi-automated builds, too.

> Couldn't we instead ask these to pass an argument to configure such that
> will make sure their requirements are satisfied, while optimizing for
> most who aren't doing parallel builds, or can take the negligible risk of
> cache corruption, or don't care about potential loss of sharing of cache
> results, but who want to benefit from the ability to configure only the
> packages that matter for them.  Are we really doing the right trade off?

I agree that this is a trade-off.  Obviously we disagree about what the
right trade-off is.

I think I've aired my viewpoint enough (it's kind of a simple view)
so I'll rest here.

Michael C

Re: (toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

[Alexandre Oliva]

On Dec 28, 2002, Michael Elizabeth Chastain <mec@shout.net> wrote:

> To me, a race is a race.  Just don't have them in the first place and no
> bugs will creep in.

We don't have them by default.  We only have them if the user chooses
to make -j.  /me thinks that, if one wants to use make -j, one should
tell configure so, such that arrangements can be made to prevent the
nasty race condition at all costs.

The cost of serializing sub-configures is that it defeats most of the
niceties of having moved sub-configures into the Makefile: why should
you have to wait for gcc's configure to complete if what you want to
build is gdb?  Or, why should I wait for all of tcl, tk, expect, etc,
to build gcc?  With serialized dependencies, one of us will lose.

Should we penalize everybody just because some might want to make -j
and make it absolutely sure that they won't have any risk whatsoever
of cache corruption and they won't miss any opportunities for caching
test results.  Couldn't we instead ask these to pass an argument to
configure such that will make sure their requirements are satisfied,
while optimizing for most who aren't doing parallel builds, or can
take the negligible risk of cache corruption, or don't care about
potential loss of sharing of cache results, but who want to benefit
from the ability to configure only the packages that matter for them.
Are we really doing the right trade off?

> I would also prefer no locks, because I don't want to deal with locking
> mechanism headaches when a user with, say, an SMP Irix NFS client and
> an HP/UX NFS server has funny build problems.

Oh, shell locking is an entirely different matter.  It's based on the
atomicity of hard-linking or directory creation, with my preference
leaning towards the latter just because it's more portable.

-- 
Alexandre Oliva   Enjoy Guarana', see http://www.ic.unicamp.br/~oliva/
Red Hat GCC Developer                 aoliva@{redhat.com, gcc.gnu.org}
CS PhD student at IC-Unicamp        oliva@{lsd.ic.unicamp.br, gnu.org}
Free Software Evangelist                Professional serial bug killer

Re: (toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

[Michael Elizabeth Chastain]

My two cents:

To me, a race is a race.  Just don't have them in the first place and no
bugs will creep in.

I would also prefer no locks, because I don't want to deal with locking
mechanism headaches when a user with, say, an SMP Irix NFS client and
an HP/UX NFS server has funny build problems.

So I'm in favor of a dumb, simple, and reliable mechanism to serialize
the configures.

Michael C

Re: (toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

[Kazu Hirata]

Hi Nathanael,

> Does this fix it?
> (If you don't have autogen, this patch is the same for Makefile.tpl and 
> Makefile.in, so just apply it to each.)

Yes.  Thank you very much!

Kazu Hirata

Re: (toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

[Alexandre Oliva]

On Dec 28, 2002, Nathanael Nerode <neroden@twcny.rr.com> wrote:

> I can't reproduce exactly the problem you had, Kazu, but I did manage to
> generate some dramatic breakage; apparently autoconf's version of
> $program_transform_name doesn't like to be after the s/x/x/ in sed.

That's just because, unlike the Cygnus configure one, autoconf's
$(program_transform_name) doesn't start with a -e.  In fact, autoconf
uses `;' as the sed command separator, whereas Cygnus configure used
to use multiple sed commands, each one prefixed by -e.  autoconf's
approach is reportedly a little bit more portable.

This unfortunately means that your fix is not quite perfect yet, since
multiple transform commands would be word-split and sed might take
them as input file names.  I suggest replacing:

        t='$(program_transform_name)'; echo as | sed -e $$t ; \

with

        echo as | sed '$(program_transform_name)'; \

that is the construct used by automake.

-- 
Alexandre Oliva   Enjoy Guarana', see http://www.ic.unicamp.br/~oliva/
Red Hat GCC Developer                 aoliva@{redhat.com, gcc.gnu.org}
CS PhD student at IC-Unicamp        oliva@{lsd.ic.unicamp.br, gnu.org}
Free Software Evangelist                Professional serial bug killer

Re: (toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

[Alexandre Oliva]

On Dec 28, 2002, Nathanael Nerode <neroden@twcny.rr.com> wrote:

> 	* Makefile.tpl: Fix dramatic bustage due to change in
> 	program_transform_name.
> 	* Makefile.in: Regenerate.

Ok

-- 
Alexandre Oliva   Enjoy Guarana', see http://www.ic.unicamp.br/~oliva/
Red Hat GCC Developer                 aoliva@{redhat.com, gcc.gnu.org}
CS PhD student at IC-Unicamp        oliva@{lsd.ic.unicamp.br, gnu.org}
Free Software Evangelist                Professional serial bug killer

(toplevel) Fix dramatic breakage for ordinary crosses (related to program_transform_name)

[Nathanael Nerode]

I can't reproduce exactly the problem you had, Kazu, but I did manage to
generate some dramatic breakage; apparently autoconf's version of
$program_transform_name doesn't like to be after the s/x/x/ in sed.
Luckily, autoconf's version of $program_transform_name is s/x/x if it would
otherwise be blank, so we can just delete the extra s/x/x.

Does this fix it?
(If you don't have autogen, this patch is the same for Makefile.tpl and 
Makefile.in, so just apply it to each.)


	* Makefile.tpl: Fix dramatic bustage due to change in
	program_transform_name.
	* Makefile.in: Regenerate.

Index: Makefile.tpl
===================================================================
RCS file: /cvs/gcc/gcc/Makefile.tpl,v
retrieving revision 1.24
diff -u -r1.24 Makefile.tpl
--- Makefile.tpl	28 Dec 2002 06:57:48 -0000	1.24
+++ Makefile.tpl	28 Dec 2002 08:21:15 -0000
@@ -282,7 +282,7 @@
     if [ '$(host_canonical)' = '$(target_canonical)' ] ; then \
       echo $(AS); \
     else \
-       t='$(program_transform_name)'; echo as | sed -e 's/x/x/' $$t ; \
+       t='$(program_transform_name)'; echo as | sed -e $$t ; \
     fi; \
   fi`
 
@@ -296,7 +296,7 @@
     if [ '$(host_canonical)' = '$(target_canonical)' ] ; then \
       echo $(LD); \
     else \
-       t='$(program_transform_name)'; echo ld | sed -e 's/x/x/' $$t ; \
+       t='$(program_transform_name)'; echo ld | sed -e $$t ; \
     fi; \
   fi`
 
@@ -308,7 +308,7 @@
     if [ '$(host_canonical)' = '$(target_canonical)' ] ; then \
       echo $(DLLTOOL); \
     else \
-       t='$(program_transform_name)'; echo dlltool | sed -e 's/x/x/' $$t ; \
+       t='$(program_transform_name)'; echo dlltool | sed -e $$t ; \
     fi; \
   fi`
 
@@ -320,7 +320,7 @@
     if [ '$(host_canonical)' = '$(target_canonical)' ] ; then \
       echo $(WINDRES); \
     else \
-       t='$(program_transform_name)'; echo windres | sed -e 's/x/x/' $$t ; \
+       t='$(program_transform_name)'; echo windres | sed -e $$t ; \
     fi; \
   fi`
 
@@ -332,7 +332,7 @@
     if [ '$(host_canonical)' = '$(target_canonical)' ] ; then \
       echo $(AR); \
     else \
-       t='$(program_transform_name)'; echo ar | sed -e 's/x/x/' $$t ; \
+       t='$(program_transform_name)'; echo ar | sed -e $$t ; \
     fi; \
   fi`
 
@@ -348,7 +348,7 @@
          echo ranlib; \
       fi; \
     else \
-       t='$(program_transform_name)'; echo ranlib | sed -e 's/x/x/' $$t ; \
+       t='$(program_transform_name)'; echo ranlib | sed -e $$t ; \
     fi; \
   fi`
 
@@ -362,7 +362,7 @@
     if [ '$(host_canonical)' = '$(target_canonical)' ] ; then \
       echo $(NM); \
     else \
-       t='$(program_transform_name)'; echo nm | sed -e 's/x/x/' $$t ; \
+       t='$(program_transform_name)'; echo nm | sed -e $$t ; \
     fi; \
   fi`