From cdb91219188278b37abc706b2c71ab9e354cf291 Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Fri, 3 Jul 2020 13:39:55 +0300
Subject: [PATCH 01/40] ci(travis): Teach Travis how to setup Lua 5.4

---
 .travis/setup_lua.sh | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.travis/setup_lua.sh b/.travis/setup_lua.sh
index a10d7cc38..470b8a83e 100755
--- a/.travis/setup_lua.sh
+++ b/.travis/setup_lua.sh
@@ -78,10 +78,13 @@ else
   elif [ "$LUA" == "lua5.3" ]; then
     curl https://www.lua.org/ftp/lua-5.3.5.tar.gz | tar xz
     cd lua-5.3.5;
+  elif [ "$LUA" == "lua5.4" ]; then
+    curl https://www.lua.org/ftp/lua-5.4.0.tar.gz | tar xz
+    cd lua-5.4.0;
   fi
 
   # Build Lua without backwards compatibility for testing
-  # perl -i -pe 's/-DLUA_COMPAT_(ALL|5_2)//' src/Makefile
+  perl -i -pe 's/-DLUA_COMPAT_\S+//' src/Makefile
   perl -i -pe 's/-DLUA_BUILD_AS_DLL/-DLUA_USE_POSIX -DLUA_DL_DLL -DLUA_BUILD_AS_DLL/' src/Makefile
   if [ "$PLATFORM" == "mingw" ]; then
     LUA_DLL=$(echo "$LUA.dll" | sed 's/\.//')
@@ -143,4 +146,6 @@ elif [ "$LUA" == "lua5.2" ]; then
   rm -rf lua-5.2.4;
 elif [ "$LUA" == "lua5.3" ]; then
   rm -rf lua-5.3.5;
+elif [ "$LUA" == "lua5.4" ]; then
+  rm -rf lua-5.4.0;
 fi

From 068fca423b67a302b84f26e8ca4b59885f962498 Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Sat, 18 Jul 2020 10:10:15 +0300
Subject: [PATCH 02/40] ci(travis): Update to LuaRocks 3.3.1 that understands
 Lua 5.4

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index e9cfdc474..fca4420f4 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -27,7 +27,7 @@ addons:
    - poppler-utils
 env:
   global:
-   - LUAROCKS=2.4.2
+   - LUAROCKS=3.3.1
    - GRAPHITE=true
    - COVERAGE=false
    - MAKEFLAGS="--jobs=4"

From f8ebd9171c0a6df34af4dbb8ad03aee67cead61b Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Fri, 3 Jul 2020 13:29:26 +0300
Subject: [PATCH 03/40] ci(travis): Add Lua 5.4 to test matrix

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index fca4420f4..e3c155cd7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -32,6 +32,7 @@ env:
    - COVERAGE=false
    - MAKEFLAGS="--jobs=4"
   jobs:
+   - LUA=lua5.4 HARFBUZZ=2.6.8
    - LUA=lua5.3 HARFBUZZ=2.6.8
    - LUA=lua5.3 HARFBUZZ=1.4.2
    - LUA=lua5.3 HARFBUZZ=1.1.3

From bc51128ae418e4f1e593f5a54e57c15b90bc735a Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Thu, 9 Jul 2020 13:05:53 +0300
Subject: [PATCH 04/40] chore(build): Enable bundling for Lua 5.4 modules

---
 Makefile-luarocks | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile-luarocks b/Makefile-luarocks
index 9af2eaa1d..da3df3434 100644
--- a/Makefile-luarocks
+++ b/Makefile-luarocks
@@ -1,4 +1,4 @@
-SUPPORTED_LUAS = 5.3 5.2 5.1
+SUPPORTED_LUAS = 5.4 5.3 5.2 5.1
 
 LUAROCKS := luarocks --tree lua_modules --lua-version $(LUA_VERSION)
 

From 652979590031bcf308fb7917773960a4846e0d24 Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Thu, 9 Jul 2020 01:47:04 +0300
Subject: [PATCH 05/40] chore(deps): Update to Lua 5.4 compatible lua-zlib

---
 sile-dev-1.rockspec | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sile-dev-1.rockspec b/sile-dev-1.rockspec
index ff5bc4f72..b72e67363 100644
--- a/sile-dev-1.rockspec
+++ b/sile-dev-1.rockspec
@@ -15,7 +15,7 @@ dependencies = {
   "cosmo == 16.06.04-1",
   "linenoise == 0.9-1",
   "lpeg == 1.0.2-1",
-  "lua-zlib == 1.2-0",
+  "lua-zlib == 1.2-1",
   "lua_cliargs == 2.3-3",
   "luaepnf == 0.3-2",
   "luaexpat == 1.3.3-1",

From db4927b103313419451539fdd8918adcf70e59cd Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Mon, 20 Jul 2020 12:10:23 +0300
Subject: [PATCH 06/40] chore(deps): Bump bundled luasec release to 0.9-1

---
 sile-dev-1.rockspec | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sile-dev-1.rockspec b/sile-dev-1.rockspec
index b72e67363..4bfbff340 100644
--- a/sile-dev-1.rockspec
+++ b/sile-dev-1.rockspec
@@ -21,7 +21,7 @@ dependencies = {
   "luaexpat == 1.3.3-1",
   "luafilesystem == 1.8.0-1",
   "luarepl == 0.9-1",
-  "luasec == 0.8.1-1",
+  "luasec == 0.9-1",
   "luasocket == 3.0rc1-2",
   "penlight == 1.7.0-1",
   "stdlib == 41.2.2-1",

From 57b28a9aaadb6a8bd2767bd8b04499360cc73231 Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Mon, 20 Jul 2020 12:31:56 +0300
Subject: [PATCH 07/40] ci(actions): Upload our dist zip not Github's broken
 one

Both options are bad. Github's directory handling makes this pretty
useless because it discards all the file permissions. Uploading the zip
like this makes for double-zipped artifacts. But at least this way I can
test what release artifacts are actually going to look like.
---
 .github/workflows/build.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 9d9538c3c..2528a0b0f 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -37,13 +37,13 @@ jobs:
           echo "::set-env name=MAKEFLAGS::-j$(nproc) -Otarget"
       - name: Make package
         run: |
-          make am__remove_distdir= dist
+          make dist
       - name: Upload Artifacts
         if: ${{ !contains(github.ref, 'refs/tags/v') }}
         uses: actions/upload-artifact@v2
         with:
           name: ${{ env.DISTDIR }}
-          path: '*${{ env.DISTDIR }}/'
+          path: ${{ env.DISTDIR }}.zip
       - name: Release
         uses: softprops/action-gh-release@v1
         if: contains(github.ref, 'refs/tags/v')

From 73e7e40a558ae3bf6a7d4284e29e1947364d855d Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Mon, 20 Jul 2020 13:15:45 +0300
Subject: [PATCH 08/40] chore(build): Make build more reproducible by sorting
 file lists

Diffing builds between different machines shouldn't be so messy.
---
 build-aux/list-dist-files.sh | 51 ++++++++++++++++++------------------
 1 file changed, 25 insertions(+), 26 deletions(-)

diff --git a/build-aux/list-dist-files.sh b/build-aux/list-dist-files.sh
index 86bfbdb42..fc21c9a21 100755
--- a/build-aux/list-dist-files.sh
+++ b/build-aux/list-dist-files.sh
@@ -3,29 +3,28 @@
 set -e
 set -o pipefail
 
-echo -n "nobase_dist_pkgdata_DATA = "
-{
-find core classes languages packages lua-libraries -type f -name '*.lua'
-find classes -type f -name '*.sil'
-} | xargs
-
-echo -ne "\nLUAMODULES = "
-find lua_modules -type f ! -name '*~' | xargs ||:
-
-echo -ne "\nLUAMODULESDIST = "
-find lua_modules_dist -type f ! -name '*~' | xargs ||:
-
-echo -ne "\nTESTSRCS ?= "
-{
-find tests -maxdepth 1 -type f -name '*.sil'
-find tests -maxdepth 1 -type f -name '*.xml'
-} | xargs
-
-echo -ne "\nTESTEXPECTS ?= "
-find tests -maxdepth 1 -type f -name '*.expected' | xargs
-
-echo -ne "\nEXAMPLESSRCS = "
-{
-find examples -maxdepth 1 -type f -name '*.sil'
-find examples/docbook -maxdepth 1 -type f -name '*.xml'
-} | xargs
+finder () {
+    echo -n ' '
+    find $@ -type f | sort | xargs echo -n ||:
+}
+
+echo -n "nobase_dist_pkgdata_DATA ="
+finder core classes languages packages lua-libraries -name '*.lua'
+finder classes -name '*.sil'
+
+echo -ne "\nLUAMODULES ="
+finder lua_modules ! -name "'*~'"
+
+echo -ne "\nLUAMODULESDIST ="
+finder lua_modules_dist ! -name "'*~'"
+
+echo -ne "\nTESTSRCS ?="
+finder tests -maxdepth 1 -name '*.sil'
+finder tests -maxdepth 1 -name '*.xml'
+
+echo -ne "\nTESTEXPECTS ?="
+finder tests -maxdepth 1 -name '*.expected'
+
+echo -ne "\nEXAMPLESSRCS ="
+finder examples -maxdepth 1 -name '*.sil'
+finder examples/docbook -maxdepth 1 -name '*.xml'

From 11efae726b307dcd63ebe99b2ba36a0cc5b512e5 Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Mon, 20 Jul 2020 13:18:01 +0300
Subject: [PATCH 09/40] chore(build): Split massive file lists into chunks

---
 build-aux/list-dist-files.sh | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/build-aux/list-dist-files.sh b/build-aux/list-dist-files.sh
index fc21c9a21..fde32cf9e 100755
--- a/build-aux/list-dist-files.sh
+++ b/build-aux/list-dist-files.sh
@@ -8,15 +8,22 @@ finder () {
     find $@ -type f | sort | xargs echo -n ||:
 }
 
-echo -n "nobase_dist_pkgdata_DATA ="
-finder core classes languages packages lua-libraries -name '*.lua'
+echo -n "nobase_dist_pkgdata_DATA = \$(LUALIBRARIES)"
+finder core classes languages packages -name '*.lua'
 finder classes -name '*.sil'
+echo -ne "\nLUALIBRARIES ="
+finder lua-libraries -name '*.lua'
 
 echo -ne "\nLUAMODULES ="
 finder lua_modules ! -name "'*~'"
 
-echo -ne "\nLUAMODULESDIST ="
-finder lua_modules_dist ! -name "'*~'"
+echo -ne "\nLUAMODULESDIST = \$(LUAMODULESDIST51) \$(LUAMODULESDIST52) \$(LUAMODULESDIST53)"
+echo -ne "\nLUAMODULESDIST51 ="
+finder lua_modules_dist -path '*5.1/*' ! -name "'*~'"
+echo -ne "\nLUAMODULESDIST52 ="
+finder lua_modules_dist -path '*5.2/*' ! -name "'*~'"
+echo -ne "\nLUAMODULESDIST53 ="
+finder lua_modules_dist -path '*5.3/*' ! -name "'*~'"
 
 echo -ne "\nTESTSRCS ?="
 finder tests -maxdepth 1 -name '*.sil'

From cb4d04bb7a83f01f0de405b20aeb3151fc08cb6c Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Mon, 20 Jul 2020 14:36:42 +0300
Subject: [PATCH 10/40] refactor(build): Stop distributing Lua modules, let
 luarocks fetch them

---
 .luacheckrc                  |  1 -
 Makefile-luarocks            | 34 ++++------------------------------
 Makefile.am                  | 21 ++++++++-------------
 build-aux/list-dist-files.sh |  8 --------
 4 files changed, 12 insertions(+), 52 deletions(-)

diff --git a/.luacheckrc b/.luacheckrc
index 64b700bad..b0ef3a5bd 100644
--- a/.luacheckrc
+++ b/.luacheckrc
@@ -11,7 +11,6 @@ exclude_files = {
   "compare-*",
   "sile-*",
   "lua_modules",
-  "lua_modules_dist",
   "lua-libraries",
   ".lua",
   ".luarocks",
diff --git a/Makefile-luarocks b/Makefile-luarocks
index 57cc365df..850532b79 100644
--- a/Makefile-luarocks
+++ b/Makefile-luarocks
@@ -1,46 +1,20 @@
-SUPPORTED_LUAS = 5.4 5.3 5.2 5.1
-
+.PHONY: installrocks
+if !SYSTEM_LUAROCKS
+_INSTALLROCKS = installrocks
 LUAROCKS := luarocks --tree lua_modules --lua-version $(LUA_VERSION)
 
-if !SYSTEM_LUAROCKS
-nobase_nodist_pkgdata_DATA = $(LUAMODULES)
 LUAMODLOCK := sile-dev-1.rockslock
 LUAMODSPEC := sile-dev-1.rockspec
 genrockslock := $(LUAROCKS) $(LUAROCKSARGS) list --porcelain | awk '{print $$1 " " $$2}'
 rocksmatch := cmp -s $(LUAMODLOCK) <($(genrockslock))
-endif
 
-.PHONY: installrocks disallow-dist-with-system-luarocks
-if !SYSTEM_LUAROCKS
 installrocks: $(LUAMODLOCK) $(shell $(rocksmatch) || echo lua_modules)
 
 lua_modules: $(LUAMODSPEC) $(shell $(rocksmatch) || echo force)
-	if test -e .git; then
-		$(LUAROCKS) $(LUAROCKSARGS) install --only-deps $<
-	else
-		rm -rf lua_modules
-		cp -a lua_modules_dist lua_modules
-		for luaver in $(filter-out $(LUA_VERSION),$(SUPPORTED_LUAS)); do
-			find lua_modules -maxdepth 3 -type d -name "*$$luaver" -execdir rm -rf {} \;
-		done
-	fi
+	$(LUAROCKS) $(LUAROCKSARGS) install --only-deps $<
 
 $(LUAMODLOCK): lua_modules $(LUAMODSPEC)
 	$(genrockslock) > $@
 
 sile: installrocks
-
-disallow-dist-with-system-luarocks: ;
-else
-disallow-dist-with-system-luarocks:
-	$(error In order to make dist you must configure --without-system-luarocks)
-	exit 1
 endif
-
-.PHONY: lua_modules_dist
-lua_modules_dist: $(foreach LUA,$(SUPPORTED_LUAS),lua_modules_dist_$(LUA))
-
-.PHONY: lua_modules_dist_%
-lua_modules_dist_%: LUAROCKS = luarocks --tree lua_modules_dist --lua-version $*
-lua_modules_dist_%: $(LUAMODSPEC) disallow-dist-with-system-luarocks
-	$(LUAROCKS) $(LUAROCKSARGS) install --only-deps $<
diff --git a/Makefile.am b/Makefile.am
index 8701eb7c5..0322903cb 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -12,15 +12,18 @@ MANUAL = documentation/sile.pdf
 Makefile-distfiles: build-aux/list-dist-files.sh $(wildcard .version sile-dev-1.rockslock)
 	$< 2>/dev/null > $@
 
+include Makefile-luarocks
+
 include Makefile-distfiles
 
+nobase_nodist_pkgdata_DATA = $(LUAMODULES)
 dist_man_MANS = sile.1
 dist_doc_DATA = README.md CHANGELOG.md
 bin_SCRIPTS = sile
 EXTRA_DIST = LICENSE spec tests examples documentation sile-dev-1.rockspec fontconfig.conf
 EXTRA_DIST += .version Makefile-distfiles
 EXTRA_DIST += build-aux/git-version-gen build-aux/list-dist-files.sh
-EXTRA_DIST += $(LUAMODULESDIST) $(MANUAL) $(EXAMPLES)
+EXTRA_DIST += $(MANUAL) $(EXAMPLES)
 
 SHELL = bash
 .ONESHELL:
@@ -36,9 +39,8 @@ BUILT_SOURCES = .version
 	cmp -s $@{,-prev} || autoreconf configure.ac --force -W none
 	sed -i -e '/rm -f/s/ core / /' configure aclocal.m4 ||:
 
-dist-hook: lua_modules_dist | $(MANUAL) $(EXAMPLES)
+dist-hook: $(MANUAL) $(EXAMPLES)
 	echo $(VERSION) > $(distdir)/.tarball-version
-	cp -fa $< $(distdir)/
 	sed -i -e '/rm -f/s/ core / /' $(distdir)/configure $(distdir)/aclocal.m4 ||:
 
 _DATA_HOOK_DEPS =
@@ -82,7 +84,7 @@ release-preview:
 .PHONY: release
 release: tagrelease
 
-dist: examples docs all lua_modules_dist sile-$(VERSION).pdf sile-$(VERSION).md
+dist: examples docs all sile-$(VERSION).pdf sile-$(VERSION).md
 
 sile-$(VERSION).pdf: $(MANUAL)
 	cp $< $@
@@ -135,7 +137,7 @@ luacheck:
 	luacheck -j$(shell nproc) -q .
 
 .PHONY: busted
-busted: | $(_INSTALL) all
+busted: | $(_INSTALLROCKS) all
 	set -f; IFS=';'
 if SYSTEM_LUAROCKS
 	packagecpath=(./{,core/}?.$(SHARED_LIB_EXT))
@@ -171,14 +173,11 @@ define runsile =
 endef
 
 
-if !SYSTEM_LUAROCKS
-_INSTALL = installrocks
-endif
 _FORCED = $(and $(SILE_COVERAGE)$(CLEAN),force)
 _TEST_DEPS = $(and $(filter tests/%,$@),$(addprefix .fonts/,$(TESTFONTFILES)))
 _DOCS_DEPS = $(and $(filter docs/%,$@),$(addprefix .fonts/,$(DOCSFONTFILES)))
 _EXAM_DEPS = $(and $(filter examples/%,$@),$(addprefix .fonts/,$(EXAMFONTFILES)))
-patterndeps = $(_FORCED) $(_TEST_DEPS) $(_EXAM_DEPS) $(_DOCS_DEPS) | $(DEPDIRS) $(_INSTALL) all
+patterndeps = $(_FORCED) $(_TEST_DEPS) $(_EXAM_DEPS) $(_DOCS_DEPS) | $(DEPDIRS) $(_INSTALLROCKS) all
 
 %.pdf: %.sil $$(patterndeps)
 	$(runsile)
@@ -241,7 +240,6 @@ $(1)-$(2):
 	[[ -d libtexpdf ]] && rmdir libtexpdf
 	[[ -h libtexpdf ]] || ln -s ../libtexpdf
 	[[ -h lua_modules ]] || ln -s ../lua_modules
-	[[ -h lua_modules_dist ]] || ln -s ../lua_modules_dist
 	[[ -h node_modules ]] || ln -s ../node_modules
 	[[ -h .fonts ]] || ln -s ../.fonts
 	./bootstrap.sh
@@ -330,9 +328,6 @@ include $(wildcard $(DEPFILES))
 # Actual rules for downloading test fonts are in a separate file
 include Makefile-fonts
 
-# All LuaRocks handling stuff consolidated
-include Makefile-luarocks
-
 CLEANFILES = $(bin_SCRIPTS) $(dist_man_MANS) $(DEPFILES) $(ACTUALS) $(TESTPDFS) $(MANUAL) Makefile-distfiles
 
 .PHONY: docker
diff --git a/build-aux/list-dist-files.sh b/build-aux/list-dist-files.sh
index fde32cf9e..0ee8a4f1a 100755
--- a/build-aux/list-dist-files.sh
+++ b/build-aux/list-dist-files.sh
@@ -17,14 +17,6 @@ finder lua-libraries -name '*.lua'
 echo -ne "\nLUAMODULES ="
 finder lua_modules ! -name "'*~'"
 
-echo -ne "\nLUAMODULESDIST = \$(LUAMODULESDIST51) \$(LUAMODULESDIST52) \$(LUAMODULESDIST53)"
-echo -ne "\nLUAMODULESDIST51 ="
-finder lua_modules_dist -path '*5.1/*' ! -name "'*~'"
-echo -ne "\nLUAMODULESDIST52 ="
-finder lua_modules_dist -path '*5.2/*' ! -name "'*~'"
-echo -ne "\nLUAMODULESDIST53 ="
-finder lua_modules_dist -path '*5.3/*' ! -name "'*~'"
-
 echo -ne "\nTESTSRCS ?="
 finder tests -maxdepth 1 -name '*.sil'
 finder tests -maxdepth 1 -name '*.xml'

From da02ea990bafe009b6e274b4dd3b7f59d7538d7f Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Mon, 20 Jul 2020 17:27:54 +0300
Subject: [PATCH 11/40] refactor(build): Move all automake patching to one
 script

---
 Makefile.am                  |  4 ++--
 bootstrap.sh                 |  2 +-
 build-aux/decore-automake.sh | 15 +++++++++++++++
 configure.ac                 |  3 +--
 4 files changed, 19 insertions(+), 5 deletions(-)
 create mode 100755 build-aux/decore-automake.sh

diff --git a/Makefile.am b/Makefile.am
index 0322903cb..0e2169f79 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -37,11 +37,11 @@ BUILT_SOURCES = .version
 	mv $@{,-prev} 2>/dev/null || touch $@-prev
 	( test -e .git && ./build-aux/git-version-gen .tarball-version || echo $(VERSION) ) > $@
 	cmp -s $@{,-prev} || autoreconf configure.ac --force -W none
-	sed -i -e '/rm -f/s/ core / /' configure aclocal.m4 ||:
+	build-aux/decore-automake.sh
 
 dist-hook: $(MANUAL) $(EXAMPLES)
 	echo $(VERSION) > $(distdir)/.tarball-version
-	sed -i -e '/rm -f/s/ core / /' $(distdir)/configure $(distdir)/aclocal.m4 ||:
+	cd $(distdir) && build-aux/decore-automake.sh
 
 _DATA_HOOK_DEPS =
 if MANUAL
diff --git a/bootstrap.sh b/bootstrap.sh
index d9d514831..ec27422f1 100755
--- a/bootstrap.sh
+++ b/bootstrap.sh
@@ -49,6 +49,6 @@ aclocal --force -W none
 automake --force-missing --add-missing -W none
 autoreconf --force -W none
 
-sed -i -e '/rm -f/s/ core / /' configure aclocal.m4 ||:
+build-aux/decore-automake.sh
 
 (cd libtexpdf; autoreconf)
diff --git a/build-aux/decore-automake.sh b/build-aux/decore-automake.sh
new file mode 100755
index 000000000..7280043b9
--- /dev/null
+++ b/build-aux/decore-automake.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+TS=$(mktemp)
+trap 'rm -rf $TS' EXIT SIGHUP SIGTERM
+
+for f in configure aclocal.m4; do
+    if [[ -f $f ]]; then
+        touch -r $f $TS
+        sed -i -e '/rm -f/s/ core / /' $f
+        touch -r $TS $f
+    fi
+done
diff --git a/configure.ac b/configure.ac
index 6f26ee46c..489fd0ad7 100644
--- a/configure.ac
+++ b/configure.ac
@@ -193,5 +193,4 @@ AC_CONFIG_FILES([Makefile src/Makefile sile.1])
 AC_CONFIG_FILES([sile], [chmod +x sile])
 
 AC_OUTPUT
-m4_esyscmd_s([test -f configure && sed -i -e '/rm -f/s/ core / /' configure])
-m4_esyscmd_s([test -f aclocal.m4 && sed -i -e '/rm -f/s/ core / /' aclocal.m4])
+m4_esyscmd_s([build-aux/decore-automake.sh])

From 9309f4a5806a1145744c24ff567cc6fa6faee6ca Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Mon, 20 Jul 2020 17:31:15 +0300
Subject: [PATCH 12/40] chore(tooling): Ignore more generated stuff

---
 .gitignore | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.gitignore b/.gitignore
index f4dc57a77..92717b1e6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
 # http://www.gnu.org/software/automake
 
+Makefile
 Makefile.in
 
 # http://www.gnu.org/software/autoconf
@@ -34,7 +35,6 @@ AUTHORS
 COPYING
 INSTALL
 ChangeLog
-Makefile
 NEWS
 README
 THANKS
@@ -72,8 +72,9 @@ lua_modules_dist
 tags
 
 # Generated resources
+Makefile-distfiles
 documentation/*.pdf
-examples/*.pdf
+examples/**/*.pdf
 gource.webm
 .fonts/*
 .sources/*

From 5e885fc2a016edf98ed8d0e223b7e4d064044209 Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Mon, 20 Jul 2020 18:18:43 +0300
Subject: [PATCH 13/40] chore(build): Normalize sort between coreutils versions

---
 build-aux/list-dist-files.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build-aux/list-dist-files.sh b/build-aux/list-dist-files.sh
index 0ee8a4f1a..73a8b6b7f 100755
--- a/build-aux/list-dist-files.sh
+++ b/build-aux/list-dist-files.sh
@@ -5,7 +5,7 @@ set -o pipefail
 
 finder () {
     echo -n ' '
-    find $@ -type f | sort | xargs echo -n ||:
+    find $@ -type f | sort -bdi | xargs echo -n ||:
 }
 
 echo -n "nobase_dist_pkgdata_DATA = \$(LUALIBRARIES)"

From 6402b3fda7d1f17d6ac05d72c764e17b0e2a9369 Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Mon, 20 Jul 2020 18:20:06 +0300
Subject: [PATCH 14/40] ci(actions): Drop cache for obsolete dist dir, cache
 lua_modules

---
 .github/workflows/build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 2528a0b0f..0c959698b 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -21,7 +21,7 @@ jobs:
         uses: actions/cache@v2
         with:
           path: |
-            lua_modules_dist
+            lua_modules
           key: luarocks-${{ hashFiles('Makefile-luarocks', 'sile-dev-1.rockspec') }}
       - name: Fetch Tags
         run: |

From 40e01f1f56602871cf40ea810d7bf506ace0a23f Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Mon, 20 Jul 2020 20:05:09 +0300
Subject: [PATCH 15/40] refactor(build): Make test target combination of
 regressions and busted

---
 .cirrus.yml  | 2 +-
 .travis.yml  | 2 +-
 Makefile.am  | 7 +++++--
 package.json | 2 +-
 4 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 9885e91a4..819194d1d 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -21,4 +21,4 @@ task:
     - ./configure --with-system-luarocks CFLAGS="-I/usr/local/include" LDFLAGS="-L/usr/local/lib"
     - gmake all
     - gmake install
-  test_script: gmake test
+  test_script: gmake regressions
diff --git a/.travis.yml b/.travis.yml
index e3c155cd7..b5c815813 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -79,7 +79,7 @@ before_script:
 script:
  - fold_unless_fail 'Building' 'build' sh -c './bootstrap.sh && ./configure && make'
  - fold_unless_fail 'Busted' 'busted'  make busted
- - fold_unless_fail 'Run regression tests' 'regression' make --output-sync test
+ - fold_unless_fail 'Run regression tests' 'regression' make --output-sync regressions
  - fold_unless_fail 'Compile documentation and examples' 'compile_docs' make --output-sync docs examples
 after_success:
  - $COVERAGE && travis_wait make coverage
diff --git a/Makefile.am b/Makefile.am
index 0e2169f79..6ea3e170e 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -121,10 +121,13 @@ selfcheck: all
 	./sile -o $$output - <<< "<sile>foo</sile>"
 	pdfinfo $$output | grep SILE
 
-.PHONY: test
-test: $(TESTSRCS) $(ACTUALS)
+.PHONY: regressions
+regressions: $(TESTSRCS) $(ACTUALS)
 	$(LOCALTESTFONTS) $(REGRESSIONSCRIPT) $(TESTSRCS)
 
+.PHONY: test
+test: regressions busted
+
 .PHONY: lint
 lint: luacheck luarocks-lint
 
diff --git a/package.json b/package.json
index a40193db4..4bc29d495 100644
--- a/package.json
+++ b/package.json
@@ -4,7 +4,7 @@
   "description": "SILE Typesetter",
   "main": "sile",
   "scripts": {
-    "test": "make test && make busted",
+    "test": "make test",
     "release": "standard-version"
   },
   "repository": {

From c3c89eb8102907e91273544214f9931a1b1ccde0 Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Mon, 20 Jul 2020 20:05:09 +0300
Subject: [PATCH 16/40] chore(build): Rebuild manual and other PDFs on version
 bump

---
 Makefile.am | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile.am b/Makefile.am
index 6ea3e170e..1e6b8fb5c 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -180,7 +180,7 @@ _FORCED = $(and $(SILE_COVERAGE)$(CLEAN),force)
 _TEST_DEPS = $(and $(filter tests/%,$@),$(addprefix .fonts/,$(TESTFONTFILES)))
 _DOCS_DEPS = $(and $(filter docs/%,$@),$(addprefix .fonts/,$(DOCSFONTFILES)))
 _EXAM_DEPS = $(and $(filter examples/%,$@),$(addprefix .fonts/,$(EXAMFONTFILES)))
-patterndeps = $(_FORCED) $(_TEST_DEPS) $(_EXAM_DEPS) $(_DOCS_DEPS) | $(DEPDIRS) $(_INSTALLROCKS) all
+patterndeps = $(_FORCED) $(_TEST_DEPS) $(_EXAM_DEPS) $(_DOCS_DEPS) sile | $(DEPDIRS) $(_INSTALLROCKS) all
 
 %.pdf: %.sil $$(patterndeps)
 	$(runsile)

From f12d8e09c5a97281ebbb640026b2729b4794941e Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Tue, 21 Jul 2020 14:54:42 +0300
Subject: [PATCH 17/40] chore(build): Add automake patch to dist file list,
 fixup da02ea99

---
 Makefile.am | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile.am b/Makefile.am
index 1e6b8fb5c..95fc073b3 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -22,7 +22,7 @@ dist_doc_DATA = README.md CHANGELOG.md
 bin_SCRIPTS = sile
 EXTRA_DIST = LICENSE spec tests examples documentation sile-dev-1.rockspec fontconfig.conf
 EXTRA_DIST += .version Makefile-distfiles
-EXTRA_DIST += build-aux/git-version-gen build-aux/list-dist-files.sh
+EXTRA_DIST += build-aux/decore-automake.sh build-aux/git-version-gen build-aux/list-dist-files.sh
 EXTRA_DIST += $(MANUAL) $(EXAMPLES)
 
 SHELL = bash

From 9ac28382beb226785f574f89353f7acb720fb949 Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Wed, 22 Jul 2020 13:15:04 +0300
Subject: [PATCH 18/40] feat(core): Allow adding --debug flag multiple times

---
 core/sile.lua | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/core/sile.lua b/core/sile.lua
index ca9b34753..245cc755e 100644
--- a/core/sile.lua
+++ b/core/sile.lua
@@ -133,7 +133,13 @@ Options:
     SILE.backend = opts.backend
   end
   if opts.debug then
-    for _, v in ipairs(std.string.split(opts.debug, ",")) do SILE.debugFlags[v] = true end
+    if type(opts.debug) ~= "table" then opts.debug = { opts.debug } end
+    for _, value in ipairs(opts.debug) do
+        SU.dump(value)
+      for _, flag in ipairs(std.string.split(value, ",")) do
+        SILE.debugFlags[flag] = true
+      end
+    end
   end
   if opts.evaluate then
     local statements = type(opts.evaluate) == "table" and opts.evaluate or { opts.evaluate }

From d5b4556ae0b9ada3b057672df3bc38c3fa9db07c Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Wed, 22 Jul 2020 13:19:51 +0300
Subject: [PATCH 19/40] docs(manual): Mention new way to add multiple debug
 flags

---
 core/sile.lua                | 2 +-
 documentation/c12-tricks.sil | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/core/sile.lua b/core/sile.lua
index 245cc755e..d2be31298 100644
--- a/core/sile.lua
+++ b/core/sile.lua
@@ -108,7 +108,7 @@ as the input file with the extension changed to .pdf.
 Options:
 
   -b, --backend=VALUE      choose an alternative output backend
-  -d, --debug=VALUE        debug SILE's operation
+  -d, --debug=VALUE        show debug information for tagged aspects of SILE's operation
   -e, --evaluate=VALUE     evaluate some Lua code before processing file
   -f, --fontmanager=VALUE  choose an alternative font manager
   -m, --makedeps=[FILE]    generate a list of dependencies in Makefile format
diff --git a/documentation/c12-tricks.sil b/documentation/c12-tricks.sil
index a12b6762c..504d85701 100644
--- a/documentation/c12-tricks.sil
+++ b/documentation/c12-tricks.sil
@@ -443,7 +443,7 @@ to get further information about what SILE is up to. SILE has a variety of
 debugging switches that can be turned on by the command line or by Lua code.
 
 Running SILE with the \code{--debug \em{facility}} switch will turn on debugging
-for a particular area of SILE’s operation:
+for a particular area or areas of SILE’s operation:
 
 \noindent{}• \code{typesetter} provides general debugging for the typesetter:
 turning characters into boxes, boxes into lines, lines into paragraphs, and
@@ -465,9 +465,9 @@ algorithm.
 
 \noindent{}• Any package or other area of SILE’s operation may define their own debugging facility; the \code{insertions} package does this, as does the Uyghur language support system (\code{--debug uyghur}).
 
-Multiple facilities can be turned on by separating them with commas:
-\code{--debug typesetter,break} will turn on debugging information for the
-typesetter and line breaker.
+Multiple facilities can be turned on by adding the flag multiple times or by
+separating them with commas: \code{--debug typesetter,break} will turn on
+debugging information for the typesetter and line breaker.
 
 From Lua, you can add entries to the \code{SILE.debugFlags} table to turn on
 debugging for a given facility. This can be useful for temporarily debugging

From e3e10ef0307068710539dd632e3d4bafa13d5633 Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Wed, 22 Jul 2020 13:23:48 +0300
Subject: [PATCH 20/40] refactor(packages): Rename debug flag to match package
 name

---
 packages/chapterverse.lua | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/packages/chapterverse.lua b/packages/chapterverse.lua
index 054eaad2d..7cb4c66e5 100644
--- a/packages/chapterverse.lua
+++ b/packages/chapterverse.lua
@@ -3,58 +3,58 @@ SILE.require("packages/infonode")
 SILE.scratch.chapterverse = {}
 
 SILE.registerCommand("save-book-title", function (_, content)
-  SU.debug("bcv", "book: " .. content[1])
+  SU.debug("chapterverse", "book: " .. content[1])
   SILE.scratch.chapterverse.book = content[1]
 end)
 
 SILE.registerCommand("save-chapter-number", function (_, content)
-  SU.debug("bcv", "chapter: " .. content[1])
+  SU.debug("chapterverse", "chapter: " .. content[1])
   SILE.scratch.chapterverse.chapter = content[1]
 end)
 
 SILE.registerCommand("save-verse-number", function (_, content)
-  SU.debug("bcv", "verse: " .. content[1])
+  SU.debug("chapterverse", "verse: " .. content[1])
   SILE.scratch.chapterverse.verse = content[1]
   local ref = {
     book = SILE.scratch.chapterverse.book,
     chapter = SILE.scratch.chapterverse.chapter,
     verse = SILE.scratch.chapterverse.verse
   }
-  SU.debug("bcv", "ref: " .. ref)
+  SU.debug("chapterverse", "ref: " .. ref)
   SILE.call("info", { category = "references", value = ref }, {})
 end)
 
 SILE.registerCommand("first-reference", function (_, _)
   local refs = SILE.scratch.info.thispage.references
-  SU.debug("bcv", "first-reference: " .. SILE.scratch.info)
+  SU.debug("chapterverse", "first-reference: " .. SILE.scratch.info)
   if refs then
-    SU.debug("bcv", "first-reference: " .. refs[1])
+    SU.debug("chapterverse", "first-reference: " .. refs[1])
     SILE.call("format-reference", {}, refs[1])
   else
-    SU.debug("bcv", "first-reference: none")
+    SU.debug("chapterverse", "first-reference: none")
   end
 end)
 
 SILE.registerCommand("last-reference", function (options, _)
   local refs = SILE.scratch.info.thispage.references
   if refs then
-    SU.debug("bcv", "last-reference: " .. refs[#(refs)])
+    SU.debug("chapterverse", "last-reference: " .. refs[#(refs)])
     SILE.call("format-reference", options, refs[#(refs)])
   else
-    SU.debug("bcv", "last-reference: none")
+    SU.debug("chapterverse", "last-reference: none")
   end
 end)
 
 SILE.registerCommand("format-reference", function (options, content)
   if type(options.showbook) == "nil" then options.showbook = true end
-  SU.debug("bcv", "formatting: " .. content)
+  SU.debug("chapterverse", "formatting: " .. content)
   local ref
   if content.book and options.showbook then
     ref = content.book .. " " .. content.chapter .. ":" .. content.verse
   else
     ref = content.chapter .. ":" .. content.verse
   end
-  SU.debug("bcv", "formatting: " .. ref)
+  SU.debug("chapterverse", "formatting: " .. ref)
   SILE.typesetter:typeset(ref)
 end)
 

From b3694b5d6668d98f522a0e8a74b6bd39cd23dbb4 Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Wed, 22 Jul 2020 13:42:56 +0300
Subject: [PATCH 21/40] docs(manual): Add documentation for all used debug
 flags

---
 documentation/c12-tricks.sil | 55 ++++++++++++++++++++++++++++--------
 1 file changed, 43 insertions(+), 12 deletions(-)

diff --git a/documentation/c12-tricks.sil b/documentation/c12-tricks.sil
index 504d85701..044b82a67 100644
--- a/documentation/c12-tricks.sil
+++ b/documentation/c12-tricks.sil
@@ -442,28 +442,59 @@ When you are experimenting with SILE and its API, you may find it necessary
 to get further information about what SILE is up to. SILE has a variety of
 debugging switches that can be turned on by the command line or by Lua code.
 
-Running SILE with the \code{--debug \em{facility}} switch will turn on debugging
-for a particular area or areas of SILE’s operation:
+Running SILE with the \code{--debug \em{facility}} switch will turn on
+debugging for a particular area or areas of SILE’s operation:
 
-\noindent{}• \code{typesetter} provides general debugging for the typesetter:
-turning characters into boxes, boxes into lines, lines into paragraphs, and
-paragraphs into pages.
+% Scan code for currently used flags:
+% rg -I '.*SU.debug(ging)?\("(\w+)".*' -r '$2' | =sort -u
+
+\noindent{}• \code{ast} provides information about how SILE parsed the document
+into an abstract syntax tree.
+
+\noindent{}• \code{break} provides (copious) information about the line
+breaking algorithm.
+
+\noindent{}• \code{fonts} shows what font families and attributes were
+attempted and what font files were used to supply them.
+
+\noindent{}• \code{frames} draws red boxes around the frames on the page.
+
+\noindent{}• \code{hboxes} draws red boxes around all the hboxes on the page.
 
 \noindent{}• \code{pagebuilder} helps to debug problems when determining page
 breaks.
 
-\noindent{}• \code{vboxes} provides even more information about page break decisions, showing you what vboxes were in SILE's queue when considering the breaking.
+\noindent{}• \code{macros} notes when new functions are defined from TeXlike
+macro syntax.
+
+\noindent{}• \code{makedeps} lists resources that were determined to be
+dependencies (use with \code{-m}).
+
+\noindent{}• \code{profile} turns on Lua profiling, which gives a report on
+where the Lua interpreter is spending its time while processing your document.
+It also makes SILE go really, really slow.
+
+\noindent{}• \code{pushback} notes how already-shaped content that didn't fit
+in frames is processed as it migrates to following ones.
 
-\noindent{}• \code{break} provides (copious) information about the line breaking
-algorithm.
+\noindent{}• \code{tokenizer} shows how input content gets broken up into
+segments before shaping.
 
-\noindent{}• \code{frames} puts red boxes around the frames on the page.
+\noindent{}• \code{typesetter} provides general debugging for the typesetter:
+turning characters into boxes, boxes into lines, lines into paragraphs, and
+paragraphs into pages.
 
-\noindent{}• \code{profile} turns on Lua profiling, which gives a report on where the Lua interpreter is spending its time while processing your document. It also makes SILE go really, really slow.
+\noindent{}• \code{vboxes} provides even more information about page break
+decisions, showing you what vboxes were in SILE's queue when considering the
+breaking.
 
-\noindent{}• \code{versions} gives a report on the versions of libraries and fonts in use; please include this information when reporting bugs in SILE!
+\noindent{}• \code{versions} gives a report on the versions of libraries and
+fonts in use; please include this information when reporting bugs in SILE!
 
-\noindent{}• Any package or other area of SILE’s operation may define their own debugging facility; the \code{insertions} package does this, as does the Uyghur language support system (\code{--debug uyghur}).
+\noindent{}• Any package or other area of SILE’s operation may define their own
+debugging tags; the \code{insertions} package does this, as do the Japanese and
+Uyghur language support systems (\code{--debug uyghur}). Often the debug flag
+is the name of the package or the function.
 
 Multiple facilities can be turned on by adding the flag multiple times or by
 separating them with commas: \code{--debug typesetter,break} will turn on

From 6c0029df469e89ce809ff833a3fa631eee14f77e Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Wed, 22 Jul 2020 13:50:43 +0300
Subject: [PATCH 22/40] fix(debug): Fix math in hbox debugging

---
 core/libtexpdf-output.lua | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/core/libtexpdf-output.lua b/core/libtexpdf-output.lua
index cb3af417f..0946a69f4 100644
--- a/core/libtexpdf-output.lua
+++ b/core/libtexpdf-output.lua
@@ -170,14 +170,14 @@ SILE.outputters.libtexpdf = {
   debugHbox = function (hbox, scaledWidth)
     ensureInit()
     pdf.colorpush_rgb(0.8, 0.3, 0.3)
-    pdf.setrule(cursorX, cursorY+(hbox.height), scaledWidth+0.5, 0.5)
-    pdf.setrule(cursorX, cursorY, 0.5, hbox.height)
-    pdf.setrule(cursorX, cursorY, scaledWidth+0.5, 0.5)
-    pdf.setrule(cursorX+scaledWidth, cursorY, 0.5, hbox.height)
+    pdf.setrule(cursorX, cursorY+(hbox.height:tonumber()), scaledWidth:tonumber()+0.5, 0.5)
+    pdf.setrule(cursorX, cursorY, 0.5, hbox.height:tonumber())
+    pdf.setrule(cursorX, cursorY, scaledWidth:tonumber()+0.5, 0.5)
+    pdf.setrule(cursorX+scaledWidth:tonumber(), cursorY, 0.5, hbox.height:tonumber())
     if hbox.depth then
-      pdf.setrule(cursorX, cursorY-(hbox.depth), scaledWidth, 0.5)
-      pdf.setrule(cursorX+scaledWidth, cursorY-(hbox.depth), 0.5, hbox.depth)
-      pdf.setrule(cursorX, cursorY-(hbox.depth), 0.5, hbox.depth)
+      pdf.setrule(cursorX, cursorY-(hbox.depth:tonumber()), scaledWidth:tonumber(), 0.5)
+      pdf.setrule(cursorX+scaledWidth:tonumber(), cursorY-(hbox.depth:tonumber()), 0.5, hbox.depth:tonumber())
+      pdf.setrule(cursorX, cursorY-(hbox.depth:tonumber()), 0.5, hbox.depth:tonumber())
 
     end
     pdf.colorpop()

From 5aad3975cc92b2641337bd65e2919ee198fe8669 Mon Sep 17 00:00:00 2001
From: Simon Cozens <simon@simon-cozens.org>
Date: Wed, 22 Jul 2020 12:29:18 +0100
Subject: [PATCH 23/40] fix(utilities): Add sorted pairs function

---
 core/utilities.lua | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/core/utilities.lua b/core/utilities.lua
index 9038b8e47..c56e3213e 100644
--- a/core/utilities.lua
+++ b/core/utilities.lua
@@ -125,6 +125,22 @@ utilities.map = function (func, array)
   return new_array
 end
 
+utilities.sortedpairs = function (t)
+  local keyset={}
+  for k,v in pairs(t) do
+    keyset[#keyset+1]=k
+  end
+  table.sort(keyset, function(a,b)
+    if type(a) ~= type(b) then return false end
+    return a < b
+  end)
+  return coroutine.wrap(function()
+    for i =1,#keyset do
+      coroutine.yield(keyset[i], t[keyset[i]])
+    end
+  end)
+end
+
 utilities.splice = function (array, start, stop, replacement)
   local ptr = start
   local room = stop - start + 1

From a28ef06b2aa1265018078e258280fc2f9a7dc348 Mon Sep 17 00:00:00 2001
From: Simon Cozens <simon@simon-cozens.org>
Date: Wed, 22 Jul 2020 12:29:55 +0100
Subject: [PATCH 24/40] fix(packages): Use sortedpairs to avoid non-determinism

---
 packages/inputfilter.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/inputfilter.lua b/packages/inputfilter.lua
index aecf41859..d366ca974 100644
--- a/packages/inputfilter.lua
+++ b/packages/inputfilter.lua
@@ -1,6 +1,6 @@
 local function transformContent(content, transformFunction, extraArgs)
   local newContent = {}
-  for k, v in pairs(content) do
+  for k, v in SU.sortedpairs(content) do
     if type(k) == "number" then
       if type(v) == "string" then
         local transformed = transformFunction(v, content, extraArgs)

From 25843dafdfed5641c17b4274c292c2feb4975e86 Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Wed, 22 Jul 2020 16:32:34 +0300
Subject: [PATCH 25/40] style(utilities): Cleanup Lua coding and fix lint
 warning

---
 core/utilities.lua | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/core/utilities.lua b/core/utilities.lua
index c56e3213e..b76ffc8b1 100644
--- a/core/utilities.lua
+++ b/core/utilities.lua
@@ -125,18 +125,19 @@ utilities.map = function (func, array)
   return new_array
 end
 
-utilities.sortedpairs = function (t)
-  local keyset={}
-  for k,v in pairs(t) do
-    keyset[#keyset+1]=k
+-- TODO: Replace with pl.tablex.sort()?
+utilities.sortedpairs = function (input)
+  local keys = {}
+  for k, _ in pairs(input) do
+    keys[#keys+1] = k
   end
-  table.sort(keyset, function(a,b)
+  table.sort(keys, function(a, b)
     if type(a) ~= type(b) then return false end
     return a < b
   end)
   return coroutine.wrap(function()
-    for i =1,#keyset do
-      coroutine.yield(keyset[i], t[keyset[i]])
+    for i = 1, #keys do
+      coroutine.yield(keys[i], input[keys[i]])
     end
   end)
 end

From 9db0a28d5c64caf9d64200d359f477bd375469eb Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Thu, 23 Jul 2020 10:08:49 +0300
Subject: [PATCH 26/40] fix(core): Iterate on sequential data with ipairs() or
 SU.sortedpairs()

---
 core/hyphenator-liang.lua | 6 +++---
 core/nodefactory.lua      | 8 ++++----
 core/utilities.lua        | 2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/core/hyphenator-liang.lua b/core/hyphenator-liang.lua
index 49f00386b..50fbda158 100644
--- a/core/hyphenator-liang.lua
+++ b/core/hyphenator-liang.lua
@@ -47,9 +47,9 @@ local function loadPatterns(hyphenator, language)
     print("No patterns for language "..language)
     return
   end
-  for _, pattern in pairs(languageset.patterns) do addPattern(hyphenator, pattern) end
+  for _, pattern in ipairs(languageset.patterns) do addPattern(hyphenator, pattern) end
   if not languageset.exceptions then languageset.exceptions = {} end
-  for _, exception in pairs(languageset.exceptions) do
+  for _, exception in ipairs(languageset.exceptions) do
     registerException(hyphenator, exception)
   end
 end
@@ -114,7 +114,7 @@ local hyphenateNode = function (node)
     local newnodes = {}
     for j, brk in ipairs(breaks) do
       if not(brk == "") then
-        for _, newNode in pairs(SILE.shaper:createNnodes(brk, node.options)) do
+        for _, newNode in ipairs(SILE.shaper:createNnodes(brk, node.options)) do
           if newNode.is_nnode then
             newNode.parent = node
             table.insert(newnodes, newNode)
diff --git a/core/nodefactory.lua b/core/nodefactory.lua
index 155e44f8d..5b6d29629 100644
--- a/core/nodefactory.lua
+++ b/core/nodefactory.lua
@@ -304,21 +304,21 @@ nodefactory.discretionary = pl.class({
     prebreakWidth = function (self)
       if self.prebw then return self.prebw end
       self.prebw = SILE.length()
-      for _, node in pairs(self.prebreak) do self.prebw:___add(node.width) end
+      for _, node in ipairs(self.prebreak) do self.prebw:___add(node.width) end
       return self.prebw
     end,
 
     postbreakWidth = function (self)
       if self.postbw then return self.postbw end
       self.postbw = SILE.length()
-      for _, node in pairs(self.postbreak) do self.pastbw:___add(node.width) end
+      for _, node in ipairs(self.postbreak) do self.pastbw:___add(node.width) end
       return self.postbw
     end,
 
     replacementWidth = function (self)
       if self.replacew then return self.replacew end
       self.replacew = SILE.length()
-      for _, node in pairs(self.replacement) do self.replacew:___add(node.width) end
+      for _, node in ipairs(self.replacement) do self.replacew:___add(node.width) end
       return self.replacew
     end,
 
@@ -522,7 +522,7 @@ nodefactory.vbox = pl.class({
     outputYourself = function (self, typesetter, line)
       typesetter.frame:advancePageDirection(self.height)
       local initial = true
-      for _, node in pairs(self.nodes) do
+      for _, node in ipairs(self.nodes) do
         if not (initial and (node.is_glue or node.is_penalty)) then
           initial = false
           node:outputYourself(typesetter, line)
diff --git a/core/utilities.lua b/core/utilities.lua
index b76ffc8b1..8bc228a86 100644
--- a/core/utilities.lua
+++ b/core/utilities.lua
@@ -274,7 +274,7 @@ end
 -- items — assuming that the current command is taking care of itself
 utilities.subContent = function (content)
   local out = { id="stuff" }
-  for key, val in pairs(content) do
+  for key, val in utilities.sortedpairs(content) do
     if type(key) == "number" then
       out[#out+1] = val
     end

From 618007b07ae2f81d12c051ef1666a0db8b6dac2e Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Tue, 21 Jul 2020 16:44:29 +0300
Subject: [PATCH 27/40] chore(build): Don't unnecessarily bump timestamps to
 avoid rebuilds

---
 Makefile-fonts    | 1 +
 Makefile-luarocks | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/Makefile-fonts b/Makefile-fonts
index edf0e32e1..5aff822b4 100644
--- a/Makefile-fonts
+++ b/Makefile-fonts
@@ -101,6 +101,7 @@ TESTFONTFILES += SBL_Hbrw.ttf
 
 .fonts/Cormorant%: .sources/Cormorant_Install_v3.601.zip | .fonts
 	bsdtar -x -f $< -C $(dir $@) --strip-components 2 "Cormorant_Install_v3.601/1. TrueType Font Files/$(notdir $@)"
+	touch $@
 
 .fonts/CrimsonPro%: .sources/CrimsonPro.zip | .fonts
 	bsdtar -x -f $< -C $(dir $@) --strip-components 1 static/$(notdir $@)
diff --git a/Makefile-luarocks b/Makefile-luarocks
index 850532b79..a7c650689 100644
--- a/Makefile-luarocks
+++ b/Makefile-luarocks
@@ -16,5 +16,5 @@ lua_modules: $(LUAMODSPEC) $(shell $(rocksmatch) || echo force)
 $(LUAMODLOCK): lua_modules $(LUAMODSPEC)
 	$(genrockslock) > $@
 
-sile: installrocks
+sile: $(LUAMODLOCK)
 endif

From 7939970397414554c45dcfe486dc736b8fb2e4fe Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Mon, 20 Jul 2020 23:57:40 +0300
Subject: [PATCH 28/40] fix(build): Ship blank lua_modules install list in
 source packages

---
 Makefile.am | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/Makefile.am b/Makefile.am
index 95fc073b3..306e0e9c8 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -9,7 +9,7 @@ endif
 
 MANUAL = documentation/sile.pdf
 
-Makefile-distfiles: build-aux/list-dist-files.sh $(wildcard .version sile-dev-1.rockslock)
+Makefile-distfiles: build-aux/list-dist-files.sh $(wildcard .version .tarball-version sile-dev-1.rockslock)
 	$< 2>/dev/null > $@
 
 include Makefile-luarocks
@@ -40,8 +40,11 @@ BUILT_SOURCES = .version
 	build-aux/decore-automake.sh
 
 dist-hook: $(MANUAL) $(EXAMPLES)
-	echo $(VERSION) > $(distdir)/.tarball-version
-	cd $(distdir) && build-aux/decore-automake.sh
+	cd $(distdir)
+	sed -i -e '/^LUAMODULES =/s/=.*/=/' Makefile-distfiles
+	echo $(VERSION) > .tarball-version
+	build-aux/decore-automake.sh
+	sed -i -e '/^LUAMODULES/d;/^\tlua_modules/d' Makefile.in
 
 _DATA_HOOK_DEPS =
 if MANUAL

From e7f4627a8cf8e6498b7c1c22b633579644a1d72a Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Mon, 20 Jul 2020 23:57:40 +0300
Subject: [PATCH 29/40] fix(build): Touch Makefile.in to avoid automake errors

If for some reason Makefile.in is older than acloacl.a4 and other
automake generated files, it will want to regenerate it ... which
requires automake tools, and specifically wants matching versions.

This simple action (after we're done mucking around with the other
files to remove automake's handling of "core") keeps it from thinking it
needs to rebuild anything.
---
 build-aux/decore-automake.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/build-aux/decore-automake.sh b/build-aux/decore-automake.sh
index 7280043b9..801e05f70 100755
--- a/build-aux/decore-automake.sh
+++ b/build-aux/decore-automake.sh
@@ -13,3 +13,9 @@ for f in configure aclocal.m4; do
         touch -r $TS $f
     fi
 done
+
+# Makefile.in being newer than aclocal.m4 at this point *is* important, this
+# point being both dist-hook and at configure time. The former is covered by a
+# side effect of a `sed` intervention, but to cover configure time changes we
+# need it here too.
+touch Makefile.in

From ee33ff71c2d978c637c01433663ccd7baf7e8fcc Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Tue, 21 Jul 2020 23:02:55 +0300
Subject: [PATCH 30/40] feat(build): Install manual to $(pdfdir) if configure
 --with-manual

Usually this is the same as $(docdir), but it does allow it to be
defined separately.
---
 Makefile.am | 26 ++++++++------------------
 1 file changed, 8 insertions(+), 18 deletions(-)

diff --git a/Makefile.am b/Makefile.am
index 306e0e9c8..924910853 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -25,6 +25,13 @@ EXTRA_DIST += .version Makefile-distfiles
 EXTRA_DIST += build-aux/decore-automake.sh build-aux/git-version-gen build-aux/list-dist-files.sh
 EXTRA_DIST += $(MANUAL) $(EXAMPLES)
 
+if MANUAL
+pdf_DATA = $(MANUAL)
+endif
+if EXAMPLES
+nobase_pdf_DATA = $(EXAMPLES)
+endif
+
 SHELL = bash
 .ONESHELL:
 .SECONDARY:
@@ -46,24 +53,7 @@ dist-hook: $(MANUAL) $(EXAMPLES)
 	build-aux/decore-automake.sh
 	sed -i -e '/^LUAMODULES/d;/^\tlua_modules/d' Makefile.in
 
-_DATA_HOOK_DEPS =
-if MANUAL
-_DATA_HOOK_DEPS += $(MANUAL)
-endif
-
-if EXAMPLES
-_DATA_HOOK_DEPS += $(EXAMPLES)
-endif
-
-install-data-hook: Makefile-distfiles $(_DATA_HOOK_DEPS)
-	:
-if MANUAL
-	rm -f "$(DESTDIR)/$(datarootdir)/$<"
-	install -Dm644 -t "$(DESTDIR)$(docdir)" $(MANUAL)
-endif
-if EXAMPLES
-	install -Dm644 -t "$(DESTDIR)$(docdir)/examples" $(EXAMPLES)
-endif
+install-data-hook: Makefile-distfiles
 
 # Whether to force tests to run from scratch
 CLEAN ?=

From 129c20757e7ca4178d415d54c91a493388f89462 Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Tue, 21 Jul 2020 23:37:59 +0300
Subject: [PATCH 31/40] refactor(build): Cleanup how automake knows to rebuild
 distfiles list

---
 Makefile-luarocks            | 11 +++---
 Makefile.am                  | 68 +++++++++++++++++++++---------------
 bootstrap.sh                 |  2 --
 build-aux/list-dist-files.sh |  3 +-
 4 files changed, 46 insertions(+), 38 deletions(-)

diff --git a/Makefile-luarocks b/Makefile-luarocks
index a7c650689..dfdf0c733 100644
--- a/Makefile-luarocks
+++ b/Makefile-luarocks
@@ -1,10 +1,9 @@
 .PHONY: installrocks
 if !SYSTEM_LUAROCKS
-_INSTALLROCKS = installrocks
-LUAROCKS := luarocks --tree lua_modules --lua-version $(LUA_VERSION)
-
-LUAMODLOCK := sile-dev-1.rockslock
 LUAMODSPEC := sile-dev-1.rockspec
+LUAMODLOCK := sile-dev-1.rockslock
+
+LUAROCKS := luarocks --tree lua_modules --lua-version $(LUA_VERSION)
 genrockslock := $(LUAROCKS) $(LUAROCKSARGS) list --porcelain | awk '{print $$1 " " $$2}'
 rocksmatch := cmp -s $(LUAMODLOCK) <($(genrockslock))
 
@@ -15,6 +14,6 @@ lua_modules: $(LUAMODSPEC) $(shell $(rocksmatch) || echo force)
 
 $(LUAMODLOCK): lua_modules $(LUAMODSPEC)
 	$(genrockslock) > $@
-
-sile: $(LUAMODLOCK)
+else
+LUAMODLOCK :=
 endif
diff --git a/Makefile.am b/Makefile.am
index 924910853..4b8a8f9f1 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,44 +1,60 @@
 ACLOCAL_AMFLAGS = -I m4
 AUTOMAKE_OPTIONS = foreign
 
+SHELL = bash
+.ONESHELL:
+.SECONDARY:
+.SECONDEXPANSION:
+.DELETE_ON_ERROR:
+
 if SYSTEM_LIBTEXPDF
 SUBDIRS = src
 else
 SUBDIRS = libtexpdf src
 endif
 
-MANUAL = documentation/sile.pdf
+# Rules for installing LuaRocks locally as pkgdata_DATA
+include Makefile-luarocks
 
-Makefile-distfiles: build-aux/list-dist-files.sh $(wildcard .version .tarball-version sile-dev-1.rockslock)
-	$< 2>/dev/null > $@
+# Actual rules for downloading test fonts are in a separate file
+include Makefile-fonts
 
-include Makefile-luarocks
+# Since we can't use $(wildcard ...) in automake file lists, we generate a file
+# with a script that builds our dynamic file lists instead. This is tricky,
+# because if we just include the file automake will flatten this include. By
+# using $(wildcard ...) to include it (the very function we couldn't use because
+# of the race condition it creates) we actually keep it from being flattened and
+# hence evaluated when we want it to be. Since the file always exists (see
+# BUILT_SOURCES and EXTRA_DIST) this doesn't induce a race.
+include $(wildcard Makefile-distfiles)
 
-include Makefile-distfiles
+MANUAL = documentation/sile.pdf
+EXAMPLES = $(addsuffix .pdf,$(basename $(EXAMPLESSRCS)))
 
+if MANUAL
+_MANUAL = $(MANUAL)
+endif
+
+if EXAMPLES
+_EXAMPLES = $(EXAMPLES)
+endif
+
+nobase_dist_pkgdata_DATA = $(SILEDATA) $(LUALIBRARIES)
 nobase_nodist_pkgdata_DATA = $(LUAMODULES)
 dist_man_MANS = sile.1
 dist_doc_DATA = README.md CHANGELOG.md
+dist_pdf_DATA = $(_MANUAL)
+nobase_dist_pdf_DATA = $(_EXAMPLES)
 bin_SCRIPTS = sile
 EXTRA_DIST = LICENSE spec tests examples documentation sile-dev-1.rockspec fontconfig.conf
 EXTRA_DIST += .version Makefile-distfiles
 EXTRA_DIST += build-aux/decore-automake.sh build-aux/git-version-gen build-aux/list-dist-files.sh
 EXTRA_DIST += $(MANUAL) $(EXAMPLES)
 
-if MANUAL
-pdf_DATA = $(MANUAL)
-endif
-if EXAMPLES
-nobase_pdf_DATA = $(EXAMPLES)
-endif
-
-SHELL = bash
-.ONESHELL:
-.SECONDARY:
-.SECONDEXPANSION:
-.DELETE_ON_ERROR:
+Makefile-distfiles: build-aux/list-dist-files.sh $(wildcard .version .tarball-version) | $(LUAMODLOCK)
+	$< 2>/dev/null > $@
 
-BUILT_SOURCES = .version
+BUILT_SOURCES = .version Makefile-distfiles $(dist_pdf_DATA) $(nobase_dist_pdf_DATA) $(LUAMODLOCK)
 
 .version: $(shell test -e .git && awk '{print ".git/" $$2}' .git/HEAD)
 	mv $@{,-prev} 2>/dev/null || touch $@-prev
@@ -53,7 +69,7 @@ dist-hook: $(MANUAL) $(EXAMPLES)
 	build-aux/decore-automake.sh
 	sed -i -e '/^LUAMODULES/d;/^\tlua_modules/d' Makefile.in
 
-install-data-hook: Makefile-distfiles
+install-exec-hook: all
 
 # Whether to force tests to run from scratch
 CLEAN ?=
@@ -133,7 +149,7 @@ luacheck:
 	luacheck -j$(shell nproc) -q .
 
 .PHONY: busted
-busted: | $(_INSTALLROCKS) all
+busted: sile
 	set -f; IFS=';'
 if SYSTEM_LUAROCKS
 	packagecpath=(./{,core/}?.$(SHARED_LIB_EXT))
@@ -147,8 +163,6 @@ endif
 .PHONY: docs
 docs: $(MANUAL)
 
-EXAMPLES = $(addsuffix .pdf,$(basename $(EXAMPLESSRCS)))
-
 .PHONY: examples
 examples: $(EXAMPLES)
 
@@ -159,7 +173,7 @@ pages = pdfinfo $@ | awk '$$1 == "Pages:" {print $$2}' || echo 0
 silepass = $(LOCALTESTFONTS) ./sile $(SILEFLAGS) $< -o $@ && pg0=$${pg} pg=$$($(pages))
 define runsile =
 	pg0=$$($(pages)) hadtoc=$$($(hastoc))
-	mkdir -p $(DEPDIR)
+	mkdir -p $(DEPDIR)/$$(dirname $@)
 	$(silepass)
 	export -n SILE_COVERAGE
 	if $(hastoc); then
@@ -168,12 +182,11 @@ define runsile =
 	fi
 endef
 
-
 _FORCED = $(and $(SILE_COVERAGE)$(CLEAN),force)
 _TEST_DEPS = $(and $(filter tests/%,$@),$(addprefix .fonts/,$(TESTFONTFILES)))
 _DOCS_DEPS = $(and $(filter docs/%,$@),$(addprefix .fonts/,$(DOCSFONTFILES)))
 _EXAM_DEPS = $(and $(filter examples/%,$@),$(addprefix .fonts/,$(EXAMFONTFILES)))
-patterndeps = $(_FORCED) $(_TEST_DEPS) $(_EXAM_DEPS) $(_DOCS_DEPS) sile | $(DEPDIRS) $(_INSTALLROCKS) all
+patterndeps = $(_FORCED) $(_TEST_DEPS) $(_EXAM_DEPS) $(_DOCS_DEPS) sile | $(DEPDIRS) $(LUAMODLOCK)
 
 %.pdf: %.sil $$(patterndeps)
 	$(runsile)
@@ -308,7 +321,7 @@ tests/%.actual: tests/%.xml $$(patterndeps)
 DEPFILES = $(addsuffix .d,$(addprefix $(DEPDIR)/,$(basename $(TESTSRCS) $(MANUAL) $(EXAMPLES))))
 DEPDIRS = $(sort $(dir $(DEPFILES)))
 
-$(DEPDIRS):
+$(DEPDIRS): Makefile-distfiles
 	mkdir -p $@
 
 $(DEPFILES): | $(DEPDIRS)
@@ -321,9 +334,6 @@ include $(wildcard $(DEPFILES))
 .sources:
 	[[ -h .sources ]] || mkdir -p $@
 
-# Actual rules for downloading test fonts are in a separate file
-include Makefile-fonts
-
 CLEANFILES = $(bin_SCRIPTS) $(dist_man_MANS) $(DEPFILES) $(ACTUALS) $(TESTPDFS) $(MANUAL) Makefile-distfiles
 
 .PHONY: docker
diff --git a/bootstrap.sh b/bootstrap.sh
index ec27422f1..d20be6d45 100755
--- a/bootstrap.sh
+++ b/bootstrap.sh
@@ -35,8 +35,6 @@ else
     ./build-aux/git-version-gen .tarball-version > .version
 fi
 
-touch -t 197001010200 Makefile-distfiles
-
 autoreconf --install -W none
 
 # See discussion in https://github.com/sile-typesetter/sile/issues/82 and
diff --git a/build-aux/list-dist-files.sh b/build-aux/list-dist-files.sh
index 73a8b6b7f..e68072ac5 100755
--- a/build-aux/list-dist-files.sh
+++ b/build-aux/list-dist-files.sh
@@ -8,9 +8,10 @@ finder () {
     find $@ -type f | sort -bdi | xargs echo -n ||:
 }
 
-echo -n "nobase_dist_pkgdata_DATA = \$(LUALIBRARIES)"
+echo -n "SILEDATA ="
 finder core classes languages packages -name '*.lua'
 finder classes -name '*.sil'
+
 echo -ne "\nLUALIBRARIES ="
 finder lua-libraries -name '*.lua'
 

From ad548a5e0c32ef5bd99f951594a9e49161aa5941 Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Thu, 23 Jul 2020 14:36:48 +0300
Subject: [PATCH 32/40] fix(build): Correct typo in dependencies for building
 docs

---
 Makefile.am | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile.am b/Makefile.am
index 4b8a8f9f1..227058df7 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -184,7 +184,7 @@ endef
 
 _FORCED = $(and $(SILE_COVERAGE)$(CLEAN),force)
 _TEST_DEPS = $(and $(filter tests/%,$@),$(addprefix .fonts/,$(TESTFONTFILES)))
-_DOCS_DEPS = $(and $(filter docs/%,$@),$(addprefix .fonts/,$(DOCSFONTFILES)))
+_DOCS_DEPS = $(and $(filter documentation/%,$@),$(addprefix .fonts/,$(DOCSFONTFILES)))
 _EXAM_DEPS = $(and $(filter examples/%,$@),$(addprefix .fonts/,$(EXAMFONTFILES)))
 patterndeps = $(_FORCED) $(_TEST_DEPS) $(_EXAM_DEPS) $(_DOCS_DEPS) sile | $(DEPDIRS) $(LUAMODLOCK)
 

From 26d6769a32c3985d18d314cf0281663d5545e650 Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Thu, 23 Jul 2020 17:33:13 +0300
Subject: [PATCH 33/40] fix(build): Work around src/libtexpdf subdirs using
 side-effects

---
 Makefile.am | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/Makefile.am b/Makefile.am
index 227058df7..4f1f0a2ba 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -54,7 +54,7 @@ EXTRA_DIST += $(MANUAL) $(EXAMPLES)
 Makefile-distfiles: build-aux/list-dist-files.sh $(wildcard .version .tarball-version) | $(LUAMODLOCK)
 	$< 2>/dev/null > $@
 
-BUILT_SOURCES = .version Makefile-distfiles $(dist_pdf_DATA) $(nobase_dist_pdf_DATA) $(LUAMODLOCK)
+BUILT_SOURCES = .version Makefile-distfiles
 
 .version: $(shell test -e .git && awk '{print ".git/" $$2}' .git/HEAD)
 	mv $@{,-prev} 2>/dev/null || touch $@-prev
@@ -69,8 +69,6 @@ dist-hook: $(MANUAL) $(EXAMPLES)
 	build-aux/decore-automake.sh
 	sed -i -e '/^LUAMODULES/d;/^\tlua_modules/d' Makefile.in
 
-install-exec-hook: all
-
 # Whether to force tests to run from scratch
 CLEAN ?=
 
@@ -124,7 +122,7 @@ ACTUALS    = $(addsuffix   .actual,$(basename $(EXPECTEDS)))
 check: selfcheck
 
 .PHONY: selfcheck
-selfcheck: all
+selfcheck: | $(_BUILT_SUBDIRS)
 	output=$$(mktemp --suffix=.pdf)
 	trap 'rm -f $$output' EXIT SIGHUP SIGTERM
 	./sile -o $$output - <<< "<sile>foo</sile>"
@@ -186,7 +184,19 @@ _FORCED = $(and $(SILE_COVERAGE)$(CLEAN),force)
 _TEST_DEPS = $(and $(filter tests/%,$@),$(addprefix .fonts/,$(TESTFONTFILES)))
 _DOCS_DEPS = $(and $(filter documentation/%,$@),$(addprefix .fonts/,$(DOCSFONTFILES)))
 _EXAM_DEPS = $(and $(filter examples/%,$@),$(addprefix .fonts/,$(EXAMFONTFILES)))
-patterndeps = $(_FORCED) $(_TEST_DEPS) $(_EXAM_DEPS) $(_DOCS_DEPS) sile | $(DEPDIRS) $(LUAMODLOCK)
+
+# TODO: remove _BUILT_SUBDIRS hack and replace it with something sensible when
+# these subdirs don't do crazy things like copying files outside of their own trees!
+_BUILT_SUBDIRS = .built-subdirs
+_SUBDIR_TELLS = core/justenoughharfbuzz.so libtexpdf/.libs/libtexpdf.so.0.0.0
+$(_BUILT_SUBDIRS): $(_SUBDIR_TELLS)
+	touch $@
+
+$(_SUBDIR_TELLS):
+	$(MAKE) $(AM_MAKEFLAGS) all-recursive
+#	$(error Running `make install`, `make dist`, or other end-game targets before `make all` unspported.)
+
+patterndeps = $(_FORCED) $(_TEST_DEPS) $(_EXAM_DEPS) $(_DOCS_DEPS) | $(DEPDIRS) $(LUAMODLOCK) $(_BUILT_SUBDIRS)
 
 %.pdf: %.sil $$(patterndeps)
 	$(runsile)
@@ -278,7 +288,7 @@ time-%.json: benchmark-%/time.json
 	cp $< $@
 
 .PRECIOUS: time.json
-time.json: sile
+time.json: | $(_BUILT_SUBDIRS)
 	$(time_action)
 
 $(call make_worktree_rules,compare)
@@ -334,7 +344,7 @@ include $(wildcard $(DEPFILES))
 .sources:
 	[[ -h .sources ]] || mkdir -p $@
 
-CLEANFILES = $(bin_SCRIPTS) $(dist_man_MANS) $(DEPFILES) $(ACTUALS) $(TESTPDFS) $(MANUAL) Makefile-distfiles
+CLEANFILES = $(bin_SCRIPTS) $(dist_man_MANS) $(DEPFILES) $(ACTUALS) $(TESTPDFS) $(MANUAL) Makefile-distfiles $(_BUILT_SUBDIRS)
 
 .PHONY: docker
 docker: Dockerfile

From 9d5719d22ce3d5e2d9bf7b72fa4aa6c2a1950160 Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Thu, 23 Jul 2020 17:33:13 +0300
Subject: [PATCH 34/40] chore(build): Make benchmark timed action Configurable

---
 Makefile.am | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/Makefile.am b/Makefile.am
index 4f1f0a2ba..13841bb1e 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -243,11 +243,7 @@ clean-worktrees:
 			done
 	fi
 
-define time_action =
-	export TIMEFORMAT=$$'{ "real": "%R", "user": "%U", "sys": "%S" }'
-	{ time (./sile documentation/sile.sil > /dev/null 2>&1) } 2> time.json
-endef
-
+time_action ?= ./sile documentation/sile.sil
 make_worktree_rules = $(eval $(foreach SHA,$(_HEADSHA) $(_BASESHA),$(call worktree_sha,$(1),$(shell git rev-parse --short=7 $(SHA)))))
 
 define worktree_sha =
@@ -261,7 +257,9 @@ $(1)-$(2):
 	[[ -h lua_modules ]] || ln -s ../lua_modules
 	[[ -h node_modules ]] || ln -s ../node_modules
 	[[ -h .fonts ]] || ln -s ../.fonts
+	[[ -h .sources ]] || ln -s ../.sources
 	./bootstrap.sh
+	cp ../config.status .
 	./configure
 	make
 
@@ -271,9 +269,10 @@ $(1)-$(2)/%: $(1)-$(2)
 	make $$*
 
 .PRECIOUS: $(1)-$(2)/time.json
-$(1)-$(2)/time.json: $(1)-$(2) $(1)-$(2)/sile
+$(1)-$(2)/time.json: $(1)-$(2)
 	cd $$<
-	$$(time_action)
+	export TIMEFORMAT=$$'{ "real": "%R", "user": "%U", "sys": "%S" }'
+	{ time ($$(time_action) > /dev/null 2>&1) } 2> time.json
 
 endef
 
@@ -289,7 +288,8 @@ time-%.json: benchmark-%/time.json
 
 .PRECIOUS: time.json
 time.json: | $(_BUILT_SUBDIRS)
-	$(time_action)
+	export TIMEFORMAT=$$'{ "real": "%R", "user": "%U", "sys": "%S" }'
+	{ time ($(time_action) > /dev/null 2>&1) } 2> time.json
 
 $(call make_worktree_rules,compare)
 
@@ -331,7 +331,7 @@ tests/%.actual: tests/%.xml $$(patterndeps)
 DEPFILES = $(addsuffix .d,$(addprefix $(DEPDIR)/,$(basename $(TESTSRCS) $(MANUAL) $(EXAMPLES))))
 DEPDIRS = $(sort $(dir $(DEPFILES)))
 
-$(DEPDIRS): Makefile-distfiles
+$(DEPDIRS): | Makefile-distfiles
 	mkdir -p $@
 
 $(DEPFILES): | $(DEPDIRS)

From b72d6ff87c213c0f78343e265215de087cbf2048 Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Thu, 23 Jul 2020 23:17:28 +0300
Subject: [PATCH 35/40] chore(docker): Let local Docker builds reuse host's
 downloaded fonts

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 75341ef62..ee54fe468 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -15,7 +15,7 @@ WORKDIR /src
 
 RUN mkdir /pkgdir
 
-RUN git clean -dxf ||:
+RUN git clean -dxf -e .fonts -e .sources ||:
 RUN git fetch --unshallow ||:
 RUN git fetch --tags ||:
 

From 6c41fcab0637259d23566aaf92cd0edbf08f7c80 Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Thu, 23 Jul 2020 20:51:37 +0300
Subject: [PATCH 36/40] chore(build): Don't touch things automake *should* be
 doing on it's own

---
 Makefile.am | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile.am b/Makefile.am
index 13841bb1e..22b3c19ca 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -82,7 +82,7 @@ tagrelease:
 	npm run release -- $(and $(RELTYPE),--release-as $(RELTYPE))
 
 .PHONY: prerelease
-prerelease: all test busted examples docs update_libtexpdf
+prerelease: test examples docs update_libtexpdf
 
 .PHONY: release-preview
 release-preview:
@@ -91,7 +91,7 @@ release-preview:
 .PHONY: release
 release: tagrelease
 
-dist: examples docs all sile-$(VERSION).pdf sile-$(VERSION).md
+dist: sile-$(VERSION).pdf sile-$(VERSION).md
 
 sile-$(VERSION).pdf: $(MANUAL)
 	cp $< $@

From 36e189481c2e4fd918e1ee5e4a462dcdef27f25b Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Fri, 24 Jul 2020 01:13:09 +0300
Subject: [PATCH 37/40] ci(actions): Placate old automake and Docker weirdness

we have a bug somewhere either in automake itself or in our dependencies
getting from zero to dist. It works on 1.16.2, but in Docker with 1.16.1
it fails. The two options are either to drop the -j arg and run a single
thread or make before make dist. The latter being faster and also more
granular in the output, that's what we're up to.
---
 .github/workflows/build.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 0c959698b..564f97ec2 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -35,6 +35,9 @@ jobs:
           ./configure --with-manual --with-examples
           echo "::set-env name=DISTDIR::sile-$(./build-aux/git-version-gen .tarball-version)"
           echo "::set-env name=MAKEFLAGS::-j$(nproc) -Otarget"
+      - name: Make
+        run: |
+          make
       - name: Make package
         run: |
           make dist

From 712bc925dfc1601111922d4bd9089ad161867020 Mon Sep 17 00:00:00 2001
From: Simon Cozens <simon@simon-cozens.org>
Date: Thu, 23 Jul 2020 20:41:30 +0100
Subject: [PATCH 38/40] fix(packages): Combine unichar output with existing
 unshaped node

---
 packages/unichar.lua | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/packages/unichar.lua b/packages/unichar.lua
index 3b444669d..16b9b302b 100644
--- a/packages/unichar.lua
+++ b/packages/unichar.lua
@@ -1,7 +1,13 @@
 SILE.registerCommand("unichar", function(_, content)
   local cp = content[1]
   if type(cp) ~= "string" then SU.error("Bad argument to \\unicode") end
-  SILE.typesetter:typeset(SU.utf8charfromcodepoint(cp))
+  local hlist = SILE.typesetter.state.nodes
+  local char = SU.utf8charfromcodepoint(cp)
+  if #hlist > 1 and hlist[#hlist].is_unshaped then
+    hlist[#hlist].text = hlist[#hlist].text .. char
+  else
+    SILE.typesetter:typeset(char)
+  end
 end)
 
 return { documentation = [[\begin{document}

From b50fd917a5735a6c3014f642c99d5272dd0a762d Mon Sep 17 00:00:00 2001
From: Simon Cozens <simon@simon-cozens.org>
Date: Thu, 23 Jul 2020 20:41:30 +0100
Subject: [PATCH 39/40] test(packages): Test \unichar with combining accents

---
 tests/bug-979.expected | 10 ++++++++++
 tests/bug-979.sil      | 10 ++++++++++
 2 files changed, 20 insertions(+)
 create mode 100644 tests/bug-979.expected
 create mode 100644 tests/bug-979.sil

diff --git a/tests/bug-979.expected b/tests/bug-979.expected
new file mode 100644
index 000000000..673efdb27
--- /dev/null
+++ b/tests/bug-979.expected
@@ -0,0 +1,10 @@
+Set paper size 	297.6377985	419.5275636
+Begin page
+Mx 	14.8819
+My 	28.1492
+Set font 	Symbola;10;400;;normal;;LTR
+T	164	(â)
+My 	40.1492
+T	164	(â)
+End page
+Finish
diff --git a/tests/bug-979.sil b/tests/bug-979.sil
new file mode 100644
index 000000000..b471b0646
--- /dev/null
+++ b/tests/bug-979.sil
@@ -0,0 +1,10 @@
+\begin[papersize=a6]{document}
+\nofolios
+\noindent
+\script[src=packages/unichar]
+\font[family=Symbola]
+\raggedright{
+â\break
+a\unichar{0x302}
+}
+\end{document}

From 4bcc47ac34e86e30b9345637d6273b51394031dd Mon Sep 17 00:00:00 2001
From: Caleb Maclennan <caleb@alerque.com>
Date: Fri, 24 Jul 2020 11:13:53 +0300
Subject: [PATCH 40/40] chore(release): 0.10.9

---
 CHANGELOG.md      | 21 +++++++++++++++++++++
 package-lock.json |  2 +-
 package.json      |  2 +-
 3 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e725bd2c5..64222aa70 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,27 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [0.10.9](https://github.com/sile-typesetter/sile/compare/v0.10.8...v0.10.9) (2020-07-24)
+
+
+### Features
+
+* **build:** Install manual to $(pdfdir) if configure --with-manual ([ee33ff7](https://github.com/sile-typesetter/sile/commit/ee33ff71c2d978c637c01433663ccd7baf7e8fcc))
+* **core:** Allow adding --debug flag multiple times ([9ac2838](https://github.com/sile-typesetter/sile/commit/9ac28382beb226785f574f89353f7acb720fb949))
+
+
+### Bug Fixes
+
+* **build:** Correct typo in dependencies for building docs ([ad548a5](https://github.com/sile-typesetter/sile/commit/ad548a5e0c32ef5bd99f951594a9e49161aa5941))
+* **build:** Ship blank lua_modules install list in source packages ([7939970](https://github.com/sile-typesetter/sile/commit/7939970397414554c45dcfe486dc736b8fb2e4fe))
+* **build:** Touch Makefile.in to avoid automake errors ([e7f4627](https://github.com/sile-typesetter/sile/commit/e7f4627a8cf8e6498b7c1c22b633579644a1d72a))
+* **build:** Work around src/libtexpdf subdirs using side-effects ([26d6769](https://github.com/sile-typesetter/sile/commit/26d6769a32c3985d18d314cf0281663d5545e650))
+* **core:** Iterate on sequential data with ipairs() or SU.sortedpairs() ([9db0a28](https://github.com/sile-typesetter/sile/commit/9db0a28d5c64caf9d64200d359f477bd375469eb))
+* **debug:** Fix math in hbox debugging ([6c0029d](https://github.com/sile-typesetter/sile/commit/6c0029df469e89ce809ff833a3fa631eee14f77e))
+* **packages:** Combine unichar output with existing unshaped node ([712bc92](https://github.com/sile-typesetter/sile/commit/712bc925dfc1601111922d4bd9089ad161867020))
+* **packages:** Use sortedpairs to avoid non-determinism ([a28ef06](https://github.com/sile-typesetter/sile/commit/a28ef06b2aa1265018078e258280fc2f9a7dc348))
+* **utilities:** Add sorted pairs function ([5aad397](https://github.com/sile-typesetter/sile/commit/5aad3975cc92b2641337bd65e2919ee198fe8669))
+
 ### [0.10.8](https://github.com/sile-typesetter/sile/compare/v0.10.7...v0.10.8) (2020-07-18)
 
 
diff --git a/package-lock.json b/package-lock.json
index e24443bd6..cde97036f 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,6 +1,6 @@
 {
   "name": "sile",
-  "version": "0.10.8",
+  "version": "0.10.9",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
diff --git a/package.json b/package.json
index 4bc29d495..64ea1eca7 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "sile",
-  "version": "0.10.8",
+  "version": "0.10.9",
   "description": "SILE Typesetter",
   "main": "sile",
   "scripts": {