From f41f0d52b50567becd8a6293fa263f2d169c4e76 Mon Sep 17 00:00:00 2001 From: Andrew Chadwick Date: Tue, 1 Nov 2016 05:15:28 +0000 Subject: [PATCH] g-i-parser: fix race condition (#1881) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Remove rmtree debugging patch It is shadowing exceptions thrown from the main "try" block of GDumpParser._execute_binary_get_tree(). There does seem to be a race condition, but it seems to be related to subprocess.check_call() and what it does on Windows. At the point xml.[blah].parse() is called, "dump.xml" is often absent! * More tolerant rmtreeing If shutil.rmtree() is failing due to some funky race condition that results in files the walk doesn't know about being left, then don't worry about it and continue. This may leave uniquely named junk lying around in the build tree, but that is by design not a problem with g-i-scanner. Just don't install the tmp-introspectXXXXXX folder by accident. * giscanner debugging: asserts and waits Make absence of dump.xml or functions.txt fatal, but only after giving them plenty of time time to appear. Yes, this is horrible code. But it fixes broken builds almost completely on my 64-bit Win7 with NTFS. * Wait for XML to be parsed completely It's possible that dump.xml might exist but be incompletely written. This nasty sleep-till-it-parses loop covers that case. Plus the usual logging so we can see if it's really needed ☺ * gobject-introspection: bump pkgrel --- .../0018-debug-rmtree-errors.mingw.patch | 26 ----------- .../0025-more-tolerant-rmtreeing.patch | 27 ++++++++++++ .../0026-giscanner-assertions-and-waits.patch | 43 +++++++++++++++++++ .../0027-wait-for-xml-parse-too.patch | 20 +++++++++ mingw-w64-gobject-introspection/PKGBUILD | 14 ++++-- 5 files changed, 100 insertions(+), 30 deletions(-) delete mode 100644 mingw-w64-gobject-introspection/0018-debug-rmtree-errors.mingw.patch create mode 100644 mingw-w64-gobject-introspection/0025-more-tolerant-rmtreeing.patch create mode 100644 mingw-w64-gobject-introspection/0026-giscanner-assertions-and-waits.patch create mode 100644 mingw-w64-gobject-introspection/0027-wait-for-xml-parse-too.patch diff --git a/mingw-w64-gobject-introspection/0018-debug-rmtree-errors.mingw.patch b/mingw-w64-gobject-introspection/0018-debug-rmtree-errors.mingw.patch deleted file mode 100644 index fb7e2d5e77..0000000000 --- a/mingw-w64-gobject-introspection/0018-debug-rmtree-errors.mingw.patch +++ /dev/null @@ -1,26 +0,0 @@ ---- gobject-introspection-1.39.3/giscanner/gdumpparser.py.orig 2014-01-11 16:55:55.000000000 +0000 -+++ gobject-introspection-1.39.3/giscanner/gdumpparser.py 2014-02-16 06:37:12.574424000 +0000 -@@ -23,6 +23,7 @@ - import tempfile - import shutil - import subprocess -+import time - from xml.etree.cElementTree import parse - - from . import ast -@@ -171,7 +172,14 @@ - return parse(out_path) - finally: - if not utils.have_debug_flag('save-temps'): -- shutil.rmtree(self._binary.tmpdir) -+ def rmtree_onerror (func, path, exinfo): -+ time.sleep (1) -+ try: -+ func (path) -+ except: -+ raise -+ -+ shutil.rmtree(self._binary.tmpdir, False, rmtree_onerror) - - # Parser - diff --git a/mingw-w64-gobject-introspection/0025-more-tolerant-rmtreeing.patch b/mingw-w64-gobject-introspection/0025-more-tolerant-rmtreeing.patch new file mode 100644 index 0000000000..738e17fcf0 --- /dev/null +++ b/mingw-w64-gobject-introspection/0025-more-tolerant-rmtreeing.patch @@ -0,0 +1,27 @@ +diff -rU2 gobject-introspection-1.50.0.orig/giscanner/dumper.py gobject-introspection-1.50.0/giscanner/dumper.py +--- gobject-introspection-1.50.0.orig/giscanner/dumper.py 2016-07-29 06:32:43.000000000 +0100 ++++ gobject-introspection-1.50.0/giscanner/dumper.py 2016-10-30 22:48:11.873424500 +0000 +@@ -168,5 +168,5 @@ + except CompilerError as e: + if not utils.have_debug_flag('save-temps'): +- shutil.rmtree(tmpdir) ++ shutil.rmtree(tmpdir, True) + raise SystemExit('compilation of temporary binary failed:' + str(e)) + +@@ -175,5 +175,5 @@ + except LinkerError as e: + if not utils.have_debug_flag('save-temps'): +- shutil.rmtree(tmpdir) ++ shutil.rmtree(tmpdir, True) + raise SystemExit('linking of temporary binary failed: ' + str(e)) + +diff -rU2 gobject-introspection-1.50.0.orig/giscanner/gdumpparser.py gobject-introspection-1.50.0/giscanner/gdumpparser.py +--- gobject-introspection-1.50.0.orig/giscanner/gdumpparser.py 2016-07-29 06:32:43.000000000 +0100 ++++ gobject-introspection-1.50.0/giscanner/gdumpparser.py 2016-10-30 22:45:58.043346300 +0000 +@@ -182,5 +182,5 @@ + finally: + if not utils.have_debug_flag('save-temps'): +- shutil.rmtree(self._binary.tmpdir) ++ shutil.rmtree(self._binary.tmpdir, True) + + # Parser diff --git a/mingw-w64-gobject-introspection/0026-giscanner-assertions-and-waits.patch b/mingw-w64-gobject-introspection/0026-giscanner-assertions-and-waits.patch new file mode 100644 index 0000000000..7d73574cd2 --- /dev/null +++ b/mingw-w64-gobject-introspection/0026-giscanner-assertions-and-waits.patch @@ -0,0 +1,43 @@ +diff -rU2 gobject-introspection-1.50.0.orig/giscanner/gdumpparser.py gobject-introspection-1.50.0/giscanner/gdumpparser.py +--- gobject-introspection-1.50.0.orig/giscanner/gdumpparser.py 2016-10-30 23:06:04.593000000 +0000 ++++ gobject-introspection-1.50.0/giscanner/gdumpparser.py 2016-10-30 23:56:17.679914000 +0000 +@@ -29,4 +29,5 @@ + import shutil + import subprocess ++import time + from xml.etree.cElementTree import parse + +@@ -151,4 +152,7 @@ + blob containing data gleaned from GObject's primitive introspection.""" + in_path = os.path.join(self._binary.tmpdir, 'functions.txt') ++ ++ assert os.path.isdir(self._binary.tmpdir), "tmpdir missing!" ++ + with open(in_path, 'w') as f: + for func in self._get_type_functions: +@@ -160,4 +164,12 @@ + f.write(func) + f.write('\n') ++ ++ for i in range(10): ++ if os.path.exists(in_path): ++ break ++ message.warn("functions.txt missing, waiting 2s (%s/10)" % (i+1,)) ++ time.sleep(2) ++ assert os.path.exists(in_path), "functions.txt still missing!" ++ + out_path = os.path.join(self._binary.tmpdir, 'dump.xml') + +@@ -179,4 +191,12 @@ + # Clean up temporaries + raise SystemExit(e) ++ ++ for i in range(10): ++ if os.path.exists(out_path): ++ break ++ message.warn("dump.xml missing, waiting 2s (%s/10)" % (i+1,)) ++ time.sleep(2) ++ assert os.path.exists(out_path), "dump.xml still missing" ++ + return parse(out_path) + finally: diff --git a/mingw-w64-gobject-introspection/0027-wait-for-xml-parse-too.patch b/mingw-w64-gobject-introspection/0027-wait-for-xml-parse-too.patch new file mode 100644 index 0000000000..09e9a45bf7 --- /dev/null +++ b/mingw-w64-gobject-introspection/0027-wait-for-xml-parse-too.patch @@ -0,0 +1,20 @@ +diff -rU2 gobject-introspection-1.50.0.orig/giscanner/gdumpparser.py gobject-introspection-1.50.0/giscanner/gdumpparser.py +--- gobject-introspection-1.50.0.orig/giscanner/gdumpparser.py 2016-10-31 00:07:38.852765600 +0000 ++++ gobject-introspection-1.50.0/giscanner/gdumpparser.py 2016-10-31 00:30:29.243390600 +0000 +@@ -199,5 +199,15 @@ + assert os.path.exists(out_path), "dump.xml still missing" + +- return parse(out_path) ++ # Just in case the XML data is not fully written at this point... ++ for i in range(10): ++ try: ++ return parse(out_path) ++ except Exception as e: ++ message.warn("XML parse failed (%d/10): %s" % (i+1, e)) ++ if i >= 10: ++ raise ++ time.sleep(2) ++ raise RuntimeError("this exception should never happen") ++ + finally: + if not utils.have_debug_flag('save-temps'): diff --git a/mingw-w64-gobject-introspection/PKGBUILD b/mingw-w64-gobject-introspection/PKGBUILD index 40ad4e726d..7fbffd45fe 100644 --- a/mingw-w64-gobject-introspection/PKGBUILD +++ b/mingw-w64-gobject-introspection/PKGBUILD @@ -6,7 +6,7 @@ pkgbase=mingw-w64-${_realname} pkgname=("${MINGW_PACKAGE_PREFIX}-${_realname}" "${MINGW_PACKAGE_PREFIX}-${_realname}-runtime") pkgver=1.50.0 -pkgrel=1 +pkgrel=2 arch=('any') url="https://live.gnome.org/GObjectIntrospection" license=("LGPL") @@ -20,29 +20,35 @@ makedepends=("${MINGW_PACKAGE_PREFIX}-gcc" # options=('!strip' 'debug') source=(https://download.gnome.org/sources/gobject-introspection/${pkgver%.*}/${_realname}-${pkgver}.tar.xz 0001-Revert-Windows-port-Work-arount-MSYS-weirdness-where.patch - 0018-debug-rmtree-errors.mingw.patch 0021-tests-no-undefined.patch 0022-change-pkg-config-invocations.mingw.patch 0024-Support-passing-arguments-to-g-ir-scanner-through-a-.all.patch + 0025-more-tolerant-rmtreeing.patch + 0026-giscanner-assertions-and-waits.patch + 0027-wait-for-xml-parse-too.patch 0055-fix-python-detection.patch) sha256sums=('1c6597c666f543c70ef3d7c893ab052968afae620efdc080c36657f4226337c5' 'f84df99e1ce5a3a0b640ef172de068958eb0b3d0be43a454dd86e6ccb8cfc386' - '095eec18d93fa2bd936eb11fca54cbd2e3992daf1a0f57cf932a31a06956dee1' '663bb39764058b1de1591765f9f465dd69eeae16fcf56b82a3692560849dacae' 'bb5f48047613216a6c3c652e0ef65c099475ffdddbea4a835b5a51279d7755ac' 'f4c2c87edf6ac4c993361fa8c5cb95f52fed5a21723148a1bc099e29702fe901' + 'f7fa69b6af7571371b1e162cacac05fc2a2877d057dd61649948d973b74a9e8d' + '19ca830262339c4ac9f21bfb8d669ee8e126a949a4237d55656e00c5ae81c451' + '3f38bdd0dd43d9cf626cbca7c2abd22a7ed0213e6756252c6d467d470d9c5948' 'f61d099aa7cd37c437f01d98bb95c57c66f07c78028675ae19fac87f943d189d') prepare() { cd ${srcdir}/${_realname}-${pkgver} patch -p1 -i "${srcdir}"/0001-Revert-Windows-port-Work-arount-MSYS-weirdness-where.patch - patch -p1 -i "${srcdir}"/0018-debug-rmtree-errors.mingw.patch patch -p1 -i "${srcdir}"/0021-tests-no-undefined.patch patch -p1 -i "${srcdir}"/0022-change-pkg-config-invocations.mingw.patch # Source for this patch is: # https://bazaar.launchpad.net/~lrn1986/stupidbuild/trunk/view/head:/src/mingw/gobject-introspection-1.0-1.44.0-2/patches/0024-Support-passing-arguments-to-g-ir-scanner-through-a-.all.patch patch -p1 -i "${srcdir}"/0024-Support-passing-arguments-to-g-ir-scanner-through-a-.all.patch + patch -p1 -i "${srcdir}"/0025-more-tolerant-rmtreeing.patch + patch -p1 -i "${srcdir}"/0026-giscanner-assertions-and-waits.patch + patch -p1 -i "${srcdir}"/0027-wait-for-xml-parse-too.patch #patch -p1 -i "${srcdir}"/0050-dont-load-msvcrt.patch patch -p1 -i "${srcdir}"/0055-fix-python-detection.patch