Be reproducible, and don't fail on examples exceptions

If example code outputs an error when run,
ipython (ipython:: blocks) now fails the whole build by default,
and tools/nbgenerate.py (.ipynb files) now leaves out that file
(leaving a broken link in the examples index).

As some examples use downloaded data and are hence expected to be
unrunnable on a Debian buildd, this instead keeps them
(::ipython with error output, .ipynb without output).

Also remove timestamps, ipykernel IDs, etc from examples output
for reproducibility.  (This is here not in debian/rules to do it
before building the search index.)

Author: Rebecca N. Palmer <rebecca_palmer@zoho.com>
Forwarded: not-needed

Gbp-Pq: Name sphinx_reproducibility_and_ignore_errors.patch

diff --git a/docs/Makefile b/docs/Makefile
index a77686dcd10e7f30df9933b6d8772f4478d1e934..622248fc5c88d5728db3522440f16567c450b966 100644 (file)
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -16,6 +16,8 @@ NOTEBOOKBUILD = nbgenerate.py
 # Internal variables.
 PAPEROPT_a4     = -D latex_paper_size=a4
 PAPEROPT_letter = -D latex_paper_size=letter
+SOURCE_DATE:=$(shell date -u -d "@$(SOURCE_DATE_EPOCH)" "+%a, %d %b %Y" || echo "xxx, xx xxx xxxx")
+SOURCE_TIME:=$(shell date -u -d "@$(SOURCE_DATE_EPOCH)" "+%T" || echo "xx:xx:xx")
 
 # Put it first so that "make" without argument is like "make help".
 help:
@@ -34,6 +36,19 @@ html:
        mkdir -p $(BUILDDIR)/source/examples/notebooks/generated
        # Black list notebooks from doc build here
        $(TOOLSPATH)$(NOTEBOOKBUILD) --parallel --report-errors --skip-existing --execute-only --execution-blacklist statespace_custom_models
+       @echo "Copying notebooks that failed execution (there are usually several in Debian because some need network and/or dependencies we don't have)"
+       cp -nav ../examples/notebooks/*.ipynb -t source/examples/notebooks/generated || true
+ifneq ($(SOURCE_DATE_EPOCH),)
+       @echo "Replacing timestamps and build paths in examples output for reproducibility"
+       for html in `find source/examples/notebooks/generated -name "*.html" -o -name "*.ipynb" -o -name "*.ipynb.txt"` ; do \
+           sed -i -e 's#$(PYTHONPATH)/statsmodels/#/usr/lib/python3/dist-packages/statsmodels/#g' \
+           -e 's# at 0x[0-9a-f]\{8,16\}\(>\|>\)# at 0xadde5de1e8ed\1#g' \
+           -e 's#tmp/ipykernel_[0-9]\+#tmp/ipykernel_nnnnnnn#g' \
+           -e 's#^\s\+.\(iopub.execute_input\|iopub.status.busy\|iopub.status.idle\|shell.execute_reply\).:.*# #g' \
+           -e 's#\(Date:.*\)[A-Z][a-z]\+, \+[0-9]\+,\? \+[A-Z][a-z]\+,\? \+[0-9]\+#\1$(SOURCE_DATE)#g' \
+           -e 's#\(Time:.*\)[0-9][0-9]:[0-9][0-9]:[0-9][0-9]#\1$(SOURCE_TIME)#g' $${html} ; \
+       done
+endif
        @echo "Running sphinx-build"
        @echo @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) $(O)
        @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) $(O)

diff --git a/docs/source/conf.py b/docs/source/conf.py
index 53ab2b440ab8026e2762d01292b63e62f86abd33..f3b25f6d6802a1d91a1c78fddb3066c1570204b8 100644 (file)
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -67,6 +67,7 @@ else:
 
 # nbsphinx options
 nbsphinx_allow_errors = True
+ipython_warning_is_error = False
 # sphinxcontrib-spelling options
 spelling_word_list_filename = ['spelling_wordlist.txt', 'names_wordlist.txt']
 spelling_ignore_pypi_package_names = True

diff --git a/examples/notebooks/stl_decomposition.ipynb b/examples/notebooks/stl_decomposition.ipynb
index d2a429b12a801e55b6d9c53e2cdc59d6a4ebacec..e8d6eeeeed0a08c887106bac2999dba882b758e5 100644 (file)
--- a/examples/notebooks/stl_decomposition.ipynb
+++ b/examples/notebooks/stl_decomposition.ipynb
@@ -570,7 +570,7 @@
    "outputs": [],
    "source": [
     "mod = STL(y, period=period, seasonal=seasonal)\n",
-    "%timeit mod.fit()\n",
+    "# removed for reproducibility: %timeit mod.fit()\n",
     "res = mod.fit()\n",
     "fig = res.plot(observed=False, resid=False)"
    ]
@@ -598,7 +598,7 @@
     "    trend_jump=trend_jump,\n",
     "    low_pass_jump=low_pass_jump,\n",
     ")\n",
-    "%timeit mod.fit()\n",
+    "# removed for reproducibility: %timeit mod.fit()\n",
     "res = mod.fit()\n",
     "fig = res.plot(observed=False, resid=False)"
    ]