From e7d29006d05055ab5b7af44078668c1915d35262 Mon Sep 17 00:00:00 2001 From: Nick Bowler Date: Tue, 15 Feb 2022 22:22:59 -0500 Subject: [PATCH] fix-gnulib: Reduce build-time impact of symbol renaming. Often a package will use more gnulib modules than actually needed by a library, with included utilities also making use of portability features provided by gnulib. With symbol renaming computed over all objects, as well as linking everything into both libraries and programs, this leads to compiling every required gnulib source 3 times, ouch! And due to how libtool convenience libraries work, the unused modules are probably wasting space in the library to boot. We can do better. We can partition the gnulib objects into two groups: those that are needed by the library (and thus require symbol renaming) and the rest which are only needed by programs. Only the first group needs any special treatment. The others can be put into an ordinary static library which is not installed. A new macro, DX_GNULIB_SYMFILES, provides the machinery to do this. --- m4/gnulib-shared.m4 | 81 ++++++++++++++++++++++++++++++++++++++ scripts/fix-gnulib.pl | 10 +++-- snippet/glconfig.mk | 12 +++++- tests/macros.at | 90 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 187 insertions(+), 6 deletions(-) create mode 100644 m4/gnulib-shared.m4 diff --git a/m4/gnulib-shared.m4 b/m4/gnulib-shared.m4 new file mode 100644 index 0000000..21b7cbb --- /dev/null +++ b/m4/gnulib-shared.m4 @@ -0,0 +1,81 @@ +dnl Copyright © 2022 Nick Bowler +dnl +dnl Hack to reduce glsym transformations to a subset of gnulib sources. +dnl +dnl License WTFPL2: Do What The Fuck You Want To Public License, version 2. +dnl This is free software: you are free to do what the fuck you want to. +dnl There is NO WARRANTY, to the extent permitted by law. + +dnl DX_GNULIB_SYMFILES(filename) +dnl +dnl When using the glconfig symbol-renaming functionality to include gnulib +dnl components in a library, it may not be the case that every file is actually +dnl used by the library. In this situation, applying the symbol renaming to +dnl every file unconditionally just makes the build take longer for no reason. +dnl +dnl Using this macro reduces the files processed for renaming to only those +dnl source files listed in the specified file, with one source file per line. +dnl This file should list every source file that could possibly be included +dnl into the library -- one way to generate such a list is by using gnulib-tool +dnl on a subset of modules. +dnl +dnl As the list depends on the results of configure, this works by installing a +dnl config.status hook to patch the gnulib_symfiles assignment in the Makefile +dnl based on the actually-enabled gnulib objects. +dnl +dnl The list of leftover objects which are not subject to symbol renaming is +dnl then placed in the gnulib_extra_objects make variable. +AC_DEFUN_ONCE([DX_GNULIB_SYMFILES], +[AC_REQUIRE([DX_PROG_JOIN])dnl +CONFIGURE_DEPENDENCIES=${CONFIGURE_DEPENDENCIES:+" "}'${top_srcdir}/$1' +AC_SUBST([CONFIGURE_DEPENDENCIES])dnl +AC_CONFIG_COMMANDS([gnulib-symfiles], +[for gl_f in $CONFIG_FILES +do + # Restrict processing to only Automake-generated makefiles. + gl_save_IFS=$IFS + IFS=:; set x $gl_f + gl_of=$[2]; gl_am=$gl_of.am # TODO fully handle user-specified input files + IFS=$gl_save_IFS + test -f "$gl_am" || test -f "$srcdir/gl_am" || continue + $MAKE -f - glconfig-objects GLCONFIG_OBJECTS="$ac_tmp/gl_objs.lst" \ + <"$gl_of" >/dev/null 2>&1 && test -f "$ac_tmp/gl_objs.lst" || continue + sed 's|[[.][^/.]*$]| &|' "$ac_tmp/gl_objs.lst" | LC_ALL=C sort -u \ + >"$ac_tmp/gl_syms.1" + LC_ALL=C sort -u >"$ac_tmp/gl_syms.2" <<'EOF' +dnl hide this include from traces so that aclocal doesn't find it. +dnl The required details for rebuild rules are handled by substituting +dnl CONFIGURE_DEPENDENCIES above. +m4_bpatsubst(m4_indir([m4_include], [$1]), [[.][^/.]*$], [ x]) +EOF + $JOIN -a1 "$ac_tmp/gl_syms.1" "$ac_tmp/gl_syms.2" >"$ac_tmp/gl_syms.3" + # Now replace default gnulib_symfiles assignment with the computed list + $AWK -f - -v f="$ac_tmp/gl_syms.3" "$gl_of" >"$ac_tmp/gl_tmp.mk" <<'EOF' +$[1] "/" $[2] "/" $[3] == "gnulib_symfiles/=/$(gnulib_all_symfiles)" { + objlst = symlst = ""; + while ((rc = getline < f) > 0) { + if ($[3] == "x") + symlst = symlst " " $[1] ".glsym"; + else + objlst = objlst " " $[1] $[2]; + } + + if (rc < 0) + exit 1; + + print "gnulib_symfiles =" symlst; + print "gnulib_extra_objects =" objlst; + next; +} +# add ourself to Automake's generated rebuild rules +$[2] == "=" && $[1] ~ /^am__(depfiles_maybe|maybe_remake_depfiles)$/ { + for (i = 3; i <= NF; i++) + if ($i == "gnulib-symfiles") + break; + $i = "gnulib-symfiles"; +} +{ print } +EOF + mv -f "$ac_tmp/gl_tmp.mk" "$gl_of" + break # should be max one makefile needing patching +done], [: "\${MAKE=${MAKE-make}}" "\${JOIN=$JOIN}"])]) diff --git a/scripts/fix-gnulib.pl b/scripts/fix-gnulib.pl index 51bdf59..69d97ca 100755 --- a/scripts/fix-gnulib.pl +++ b/scripts/fix-gnulib.pl @@ -1,6 +1,6 @@ #!/usr/bin/env perl # -# Copyright © 2011-2014, 2020-2021 Nick Bowler +# Copyright © 2011-2014, 2020-2022 Nick Bowler # # Prepare the Gnulib tree for inclusion into a non-recursive automake build. # While the output of gnulib-tool is "include"-able if the --makefile-name @@ -254,10 +254,12 @@ EOF print <<'EOF' if ($use_libtool); gnulib_lt_objects = $(libgnu_la_OBJECTS) $(gl_LTLIBOBJS) gnulib_objects = $(gnulib_lt_objects) +gnulib_all_symfiles = $(gnulib_lt_objects:.lo=.glsym) $(gnulib_objects): $(gnulib_headers) EOF print <<'EOF' if (!$use_libtool); gnulib_objects = $(libgnu_a_OBJECTS) $(gl_LIBOBJS) +gnulib_all_symfiles = $(gnulib_objects:.@OBJEXT@=.glsym) $(gnulib_objects): $(gnulib_headers) EOF @@ -271,9 +273,9 @@ AC_SUBST([GLSRC], [lib]) AC_CONFIG_LIBOBJ_DIR([lib]) AC_DEFUN_ONCE([DX_GLSYM_PREFIX], -[AC_REQUIRE([DX_AUTOMAKE_COMPAT])AC_REQUIRE([DX_EXPORTED_SH]) -AC_SUBST([GLSYM_PREFIX], [$1]) -]) +[AC_REQUIRE([DX_AUTOMAKE_COMPAT])AC_REQUIRE([DX_EXPORTED_SH])dnl +AC_SUBST([GLSYM_PREFIX], [$1])dnl +AC_SUBST([gnulib_symfiles], ['$(gnulib_all_symfiles)'])]) EOF print <<'EOF' if ($for_library); diff --git a/snippet/glconfig.mk b/snippet/glconfig.mk index 71ca880..b7ed4cc 100644 --- a/snippet/glconfig.mk +++ b/snippet/glconfig.mk @@ -1,4 +1,4 @@ -# Copyright © 2011-2013, 2019, 2021 Nick Bowler +# Copyright © 2011-2013, 2019, 2021-2022 Nick Bowler # # Automake fragment to generate a Gnulib config header to rewrite exported # symbols. This fragment relies on the Gnulib makefile postprocessing done by @@ -20,7 +20,6 @@ GLSYM_V = $(GLSYM_V_@AM_V@) GLSYM_V_ = $(GLSYM_V_@AM_DEFAULT_V@) GLSYM_V_0 = @printf ' %$(DX_ALIGN_V)s %s\n' 'GLSYM ' $<; -gnulib_symfiles = $(gnulib_lt_objects:.lo=.glsym) gnulib_headers += $(GLCONFIG) # This suffix rule triggers symbol generation only on demand. Dependencies are @@ -45,6 +44,15 @@ clean-glconfig: done .PHONY: clean-glconfig +# Produce the list of all currently-enabled gnulib object files to assist with +# external build helpers. +GLCONFIG_OBJECTS = &1 +glconfig-objects: + @:; { \ + for f in $(gnulib_objects); do echo "$$f"; done; \ + } >$(GLCONFIG_OBJECTS) +.PHONY: glconfig-objects + # The config header requires compilation of all gnulib object files via the # .glsym rule above. However, it cannot depend on those build products # directly because they are phony, and would make this header never up-to-date. diff --git a/tests/macros.at b/tests/macros.at index 3f39fbd..5202790 100644 --- a/tests/macros.at +++ b/tests/macros.at @@ -339,3 +339,93 @@ done exit 1]) AT_CLEANUP + +AT_SETUP([DX_GNULIB_SYMFILES]) +AT_KEYWORDS([DX_GNULIB_SYMFILES macro]) + +echo : >compile +TEST_CONFIGURE_AC([[AM_INIT_AUTOMAKE([foreign]) +AM_PROG_CC_C_O +AC_SUBST([GLSRC], [.]) +DX_AUTOMAKE_COMPAT +DX_GNULIB_SYMFILES([symfiles.lst]) +AC_CONFIG_FILES([Makefile]) +]]) + +AT_DATA([symfiles.lst], +[[b.foo +d.bar +]]) + +cp "$srcdir/snippet/glconfig.mk" . +AT_DATA([Makefile.am], +[[CLEANFILES = +DISTCLEANFILES = + +# automake won't emit this without sources but we expect it to be set +am__depfiles_maybe = depfiles +am__maybe_remake_depfiles = depfiles + +gnulib_headers = +gnulib_objects = a.o c.o b.o e.o d.o +gnulib_all_symfiles = $(gnulib_objects:.o=.glsym) +gnulib_symfiles = $(gnulib_all_symfiles) + +do_test: + printf '%s\n' $(gnulib_symfiles) | LC_ALL=C sort; \ + echo ///; printf '%s\n' $(gnulib_extra_objects) | LC_ALL=C sort +.PHONY: do_test + +include $(top_srcdir)/glconfig.mk +]]) +TEST_AUTORECONF + +TEST_CONFIGURE +AT_CHECK([make -s do_test], [0], [[b.glsym +d.glsym +/// +a.o +c.o +e.o +]]) + +dnl verify rebuild rule insertion +sed '/^gnulib_objects/s/b[[.]]o//' Makefile.in >tmp +mv -f tmp Makefile.in +AT_CHECK([make -s do_test >rebuild.out && + sed '/^config.status:/,$!d' rebuild.out], [0], +[[config.status: creating Makefile +config.status: executing depfiles commands +config.status: executing gnulib-symfiles commands +d.glsym +/// +a.o +c.o +e.o +]]) + +AT_CLEANUP + +AT_SETUP([DX_GNULIB_SYMFILES distribution]) +AT_KEYWORDS([DX_GNULIB_SYMFILES macro]) + +TEST_CONFIGURE_AC([[AM_INIT_AUTOMAKE([foreign]) +DX_GNULIB_SYMFILES([symfiles.lst]) +AC_CONFIG_FILES([Makefile]) +]]) + +AT_DATA([symfiles.lst]) +AT_DATA([Makefile.am], +[[foo: ; printf '%s\n' $(top_srcdir) $(DISTFILES) +]]) +TEST_AUTORECONF + +TEST_CONFIGURE +AT_CHECK([make -s foo], [0], [stdout]) +AT_CHECK([exec 3