KalmanFilter broken on armhf / hppa

Warn on use, and xfail the tests

Author: Rebecca N. Palmer <rebecca_palmer@zoho.com>
Bug-Debian: https://bugs.debian.org/924036
Bug-Ubuntu: https://launchpad.net/bugs/1819227
Forwarded: no

Gbp-Pq: Name xfail_kalman_armhf.patch

diff --git a/statsmodels/tsa/innovations/tests/test_arma_innovations.py b/statsmodels/tsa/innovations/tests/test_arma_innovations.py
index 509074bba3ea0082e3e3debca840977b0425023e..871a10e4abdb11bf6c36c57d8c7c30a44bc775d4 100644 (file)
--- a/statsmodels/tsa/innovations/tests/test_arma_innovations.py
+++ b/statsmodels/tsa/innovations/tests/test_arma_innovations.py
@@ -9,8 +9,10 @@ from numpy.testing import assert_allclose
 
 from statsmodels.tsa.innovations import arma_innovations
 from statsmodels.tsa.statespace.sarimax import SARIMAX
+from statsmodels.tsa.statespace.representation import warn_kalman
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 @pytest.mark.parametrize("ar_params,ma_params,sigma2", [
     (np.array([]), np.array([]), 1),
     (np.array([0.]), np.array([0.]), 1),

diff --git a/statsmodels/tsa/statespace/representation.py b/statsmodels/tsa/statespace/representation.py
index 3f8a60f0055b826d963a2ba2cf9407d0f29883b6..3f77297a119ec1f929bf3d40aa063debd0170576 100644 (file)
--- a/statsmodels/tsa/statespace/representation.py
+++ b/statsmodels/tsa/statespace/representation.py
@@ -12,6 +12,10 @@ from .tools import (
 )
 from .initialization import Initialization
 from . import tools
+import platform
+import warnings
+import sys
+warn_kalman = "Representation/KalmanFilter (and hence much of tsa.statespace) can give wrong results on armhf (armv7) and hppa - https://bugs.debian.org/924036" if ((platform.uname()[4].startswith('arm') or platform.uname()[4].startswith('aarch') or platform.uname()[4].startswith('hppa')) and np.log2(sys.maxsize)<=32) else False # test results at end of https://buildd.debian.org/status/fetch.php?pkg=statsmodels&arch=armhf&ver=0.8.0-8&stamp=1551488279&raw=0 ; the use of log2 rather than 2**32 is to avoid overflow on Python 2
 
 
 class OptionWrapper(object):
@@ -260,6 +264,8 @@ class Representation(object):
                  design=None, obs_intercept=None, obs_cov=None,
                  transition=None, state_intercept=None, selection=None,
                  state_cov=None, statespace_classes=None, **kwargs):
+        if warn_kalman:
+            warnings.warn(warn_kalman)
         self.shapes = {}
 
         # Check if k_endog is actually the endog array

diff --git a/statsmodels/tsa/statespace/tests/test_dynamic_factor.py b/statsmodels/tsa/statespace/tests/test_dynamic_factor.py
index c789f5d06916f283a7456ff63525005ae3ee6fa0..a4a5b46ae2d6be3e7a15cd685285e5d98b6ac45d 100644 (file)
--- a/statsmodels/tsa/statespace/tests/test_dynamic_factor.py
+++ b/statsmodels/tsa/statespace/tests/test_dynamic_factor.py
@@ -17,13 +17,15 @@ import pytest
 from statsmodels.tsa.statespace import dynamic_factor
 from .results import results_varmax, results_dynamic_factor
 from statsmodels.iolib.summary import forg
+from statsmodels.tsa.statespace.representation import warn_kalman
+import pytest
 
 current_path = os.path.dirname(os.path.abspath(__file__))
 
 output_path = os.path.join('results', 'results_dynamic_factor_stata.csv')
 output_results = pd.read_csv(os.path.join(current_path, output_path))
 
-
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 class CheckDynamicFactor(object):
     @classmethod
     def setup_class(cls, true, k_factors, factor_order, cov_type='approx',

diff --git a/statsmodels/tsa/statespace/tests/test_kalman.py b/statsmodels/tsa/statespace/tests/test_kalman.py
index a33f257ae41f55d9628d64e5623554c8ca4f1500..20fdea352e5e985c36c652d9e18435a6885991d5 100644 (file)
--- a/statsmodels/tsa/statespace/tests/test_kalman.py
+++ b/statsmodels/tsa/statespace/tests/test_kalman.py
@@ -24,6 +24,7 @@ import copy
 import numpy as np
 import pandas as pd
 import os
+from statsmodels.tsa.statespace.representation import warn_kalman
 import pytest
 
 from scipy.linalg.blas import find_best_blas_type
@@ -294,7 +295,7 @@ class TestClark1987SingleComplex(Clark1987):
             atol=1e-2
         )
 
-
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 class TestClark1987DoubleComplex(Clark1987):
     """
     Basic double precision complex test for the loglikelihood and filtered
@@ -363,7 +364,7 @@ class TestClark1987ForecastDouble(Clark1987Forecast):
         cls.model, cls.filter = cls.init_filter()
         cls.result = cls.run_filter()
 
-
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 class TestClark1987ForecastDoubleComplex(Clark1987Forecast):
     """
     Basic double complex forecasting test for the loglikelihood and filtered
@@ -654,7 +655,7 @@ class TestClark1989ForecastDouble(Clark1989Forecast):
         cls.model, cls.filter = cls.init_filter()
         cls.result = cls.run_filter()
 
-
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 class TestClark1989ForecastDoubleComplex(Clark1989Forecast):
     """
     Basic double complex forecasting test for the loglikelihood and filtered

diff --git a/statsmodels/tsa/statespace/tests/test_mlemodel.py b/statsmodels/tsa/statespace/tests/test_mlemodel.py
index 414e9edcbce9e1a514135b3d8358cee3bde1d896..8b8d11e30c5a5c5720ed051343301d12c335d268 100644 (file)
--- a/statsmodels/tsa/statespace/tests/test_mlemodel.py
+++ b/statsmodels/tsa/statespace/tests/test_mlemodel.py
@@ -9,6 +9,7 @@ from __future__ import division, absolute_import, print_function
 import os
 import re
 import warnings
+from statsmodels.tsa.statespace.representation import warn_kalman
 
 import numpy as np
 import pandas as pd
@@ -275,7 +276,7 @@ def test_score_misc():
 def test_from_formula():
     assert_raises(NotImplementedError, lambda: MLEModel.from_formula(1, 2, 3))
 
-
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 def test_score_analytic_ar1():
     # Test the score against the analytic score for an AR(1) model with 2
     # observations
@@ -444,6 +445,7 @@ def test_cov_params():
             mod.fit(res.params, disp=-1, cov_type='invalid_cov_type')
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 def test_transform():
     # The transforms in MLEModel are noops
     mod = MLEModel([1, 2], **kwargs)

diff --git a/statsmodels/tsa/statespace/tests/test_pickle.py b/statsmodels/tsa/statespace/tests/test_pickle.py
index 1615aa754c99ebf70213c9343ae812c4e96e7f1e..2a10b56c5a03253052ee53cc096cb657f7ba8249 100644 (file)
--- a/statsmodels/tsa/statespace/tests/test_pickle.py
+++ b/statsmodels/tsa/statespace/tests/test_pickle.py
@@ -17,6 +17,7 @@ from __future__ import division, absolute_import, print_function
 import numpy as np
 import pandas as pd
 from numpy.testing import assert_equal, assert_allclose
+from statsmodels.tsa.statespace.representation import warn_kalman
 import pytest
 
 from statsmodels.compat import cPickle
@@ -39,6 +40,7 @@ def data():
     return data_
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 def test_pickle_fit_sarimax(data):
     # Fit an ARIMA(1,1,0) to log GDP
     mod = sarimax.SARIMAX(data['lgdp'], order=(1, 1, 0))
@@ -54,6 +56,7 @@ def test_pickle_fit_sarimax(data):
     assert_allclose(res.impulse_responses(10), res.impulse_responses(10))
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 def test_unobserved_components_pickle():
     # Tests for missing data
     nobs = 20
@@ -86,6 +89,7 @@ def test_unobserved_components_pickle():
         assert_allclose(res.impulse_responses(10), res.impulse_responses(10))
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 def test_kalman_filter_pickle(data):
     # Construct the statespace representation
     true = results_kalman_filter.uc_uni
@@ -134,6 +138,7 @@ def test_kalman_filter_pickle(data):
                     pkl_results.filtered_state[3][true['start']:])
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 def test_representation_pickle():
     nobs = 10
     k_endog = 2

diff --git a/statsmodels/tsa/statespace/tests/test_representation.py b/statsmodels/tsa/statespace/tests/test_representation.py
index 829babe2a9c08b3581b065ca32baa47e0341c915..830444f2c54d8106d022762caf03be0d805cdcba 100644 (file)
--- a/statsmodels/tsa/statespace/tests/test_representation.py
+++ b/statsmodels/tsa/statespace/tests/test_representation.py
@@ -17,6 +17,7 @@ from __future__ import division, absolute_import, print_function
 import os
 import warnings
 
+from statsmodels.tsa.statespace.representation import warn_kalman
 import numpy as np
 import pandas as pd
 import pytest
@@ -222,7 +223,7 @@ class TestClark1987ForecastDouble(Clark1987Forecast):
         super(TestClark1987ForecastDouble, cls).setup_class()
         cls.results = cls.run_filter()
 
-
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 class TestClark1987ForecastDoubleComplex(Clark1987Forecast):
     """
     Basic double complex forecasting test for the loglikelihood and filtered
@@ -455,7 +456,7 @@ class TestClark1989ForecastDouble(Clark1989Forecast):
         super(TestClark1989ForecastDouble, cls).setup_class()
         cls.results = cls.run_filter()
 
-
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 class TestClark1989ForecastDoubleComplex(Clark1989Forecast):
     """
     Basic double complex forecasting test for the loglikelihood and filtered

diff --git a/statsmodels/tsa/statespace/tests/test_sarimax.py b/statsmodels/tsa/statespace/tests/test_sarimax.py
index a932e0216b07556ac2fb4356c6f013b6838314c8..8894e10d3f0882ee1925c4fc37dacd9fb1a46a32 100644 (file)
--- a/statsmodels/tsa/statespace/tests/test_sarimax.py
+++ b/statsmodels/tsa/statespace/tests/test_sarimax.py
@@ -11,6 +11,7 @@ import warnings
 
 from statsmodels.compat.platform import PLATFORM_WIN
 
+from statsmodels.tsa.statespace.representation import warn_kalman
 import numpy as np
 import pandas as pd
 import pytest
@@ -33,6 +34,7 @@ coverage_path = os.path.join('results', 'results_sarimax_coverage.csv')
 coverage_results = pd.read_csv(os.path.join(current_path, coverage_path))
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 class TestSARIMAXStatsmodels(object):
     """
     Test ARIMA model using SARIMAX class against statsmodels ARIMA class
@@ -126,6 +128,7 @@ class TestRealGDPARStata(object):
         )
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 class SARIMAXStataTests(object):
     def test_loglike(self):
         assert_almost_equal(
@@ -1023,6 +1026,7 @@ class SARIMAXCoverageTest(object):
         model.enforce_stationarity = stat
         model.enforce_invertibility = inv
 
+    @pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
     def test_results(self):
         self.result = self.model.filter(self.true_params)
 
@@ -1785,6 +1789,7 @@ class Test_seasonal_arma_trend_polynomial(SARIMAXCoverageTest):
         tps = cls.true_params
         cls.true_params[:2] = (1 - tps[2:5].sum()) * tps[:2]
 
+    @pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
     def test_results(self):
         self.result = self.model.filter(self.true_params)
 
@@ -1835,6 +1840,7 @@ class Test_seasonal_arma_diff_seasonal_diff(SARIMAXCoverageTest):
         super(Test_seasonal_arma_diff_seasonal_diff, cls).setup_class(
             47, *args, **kwargs)
 
+    @pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
     def test_results(self):
         self.result = self.model.filter(self.true_params)
 
@@ -2092,6 +2098,7 @@ def test_results():
     assert_almost_equal(res.maparams, [-0.5])
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 def test_misc_exog():
     # Tests for missing data
     nobs = 20

diff --git a/statsmodels/tsa/statespace/tests/test_save.py b/statsmodels/tsa/statespace/tests/test_save.py
index 5dcd6dc5dea8aae4bd8a7f8f7045042556e65dc5..19f18384b8edd9f83074eaa77b368f69f4e57301 100644 (file)
--- a/statsmodels/tsa/statespace/tests/test_save.py
+++ b/statsmodels/tsa/statespace/tests/test_save.py
@@ -8,6 +8,7 @@ from statsmodels.compat import cPickle
 import tempfile
 import os
 
+from statsmodels.tsa.statespace.representation import warn_kalman
 import pytest
 
 from statsmodels import datasets
@@ -29,6 +30,7 @@ def temp_filename():
               "{filename}.".format(filename=filename))
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 def test_sarimax(temp_filename):
     mod = sarimax.SARIMAX(macrodata['realgdp'].values, order=(4, 1, 0))
     res = mod.smooth(mod.start_params)
@@ -40,6 +42,7 @@ def test_sarimax(temp_filename):
     assert_allclose(res.llf, res2.llf)
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 def test_sarimax_pickle():
     mod = sarimax.SARIMAX(macrodata['realgdp'].values, order=(4, 1, 0))
     pkl_mod = cPickle.loads(cPickle.dumps(mod))
@@ -52,6 +55,7 @@ def test_sarimax_pickle():
     assert_allclose(res.llf, pkl_res.llf)
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 def test_structural(temp_filename):
     mod = structural.UnobservedComponents(
         macrodata['realgdp'].values, 'llevel')
@@ -64,6 +68,7 @@ def test_structural(temp_filename):
     assert_allclose(res.llf, res2.llf)
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 def test_structural_pickle():
     mod = structural.UnobservedComponents(
         macrodata['realgdp'].values, 'llevel')
@@ -77,6 +82,7 @@ def test_structural_pickle():
     assert_allclose(res.llf, pkl_res.llf)
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 def test_dynamic_factor(temp_filename):
     mod = dynamic_factor.DynamicFactor(
         macrodata[['realgdp', 'realcons']].diff().iloc[1:].values, k_factors=1,
@@ -90,6 +96,7 @@ def test_dynamic_factor(temp_filename):
     assert_allclose(res.llf, res2.llf)
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 def test_dynamic_factor_pickle(temp_filename):
     mod = varmax.VARMAX(
         macrodata[['realgdp', 'realcons']].diff().iloc[1:].values,
@@ -111,6 +118,7 @@ def test_dynamic_factor_pickle(temp_filename):
     assert_allclose(res.llf, res2.llf)
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 def test_varmax(temp_filename):
     mod = varmax.VARMAX(
         macrodata[['realgdp', 'realcons']].diff().iloc[1:].values,
@@ -124,6 +132,7 @@ def test_varmax(temp_filename):
     assert_allclose(res.llf, res2.llf)
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 def test_varmax_pickle(temp_filename):
     mod = varmax.VARMAX(
         macrodata[['realgdp', 'realcons']].diff().iloc[1:].values,

diff --git a/statsmodels/tsa/statespace/tests/test_structural.py b/statsmodels/tsa/statespace/tests/test_structural.py
index 10fa4b6687c81b301033f94886d44dfefb4d3ad1..d13d1472f7fe5054695a98958efbdac2e4c19677 100644 (file)
--- a/statsmodels/tsa/statespace/tests/test_structural.py
+++ b/statsmodels/tsa/statespace/tests/test_structural.py
@@ -13,6 +13,7 @@ import warnings
 import numpy as np
 from numpy.testing import assert_equal, assert_allclose
 import pandas as pd
+from statsmodels.tsa.statespace.representation import warn_kalman
 import pytest
 
 from statsmodels.datasets import macrodata
@@ -168,6 +169,7 @@ def test_local_linear_deterministic_trend(close_figures):
     run_ucm('local_linear_deterministic_trend')
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 def test_local_linear_trend(close_figures):
     run_ucm('local_linear_trend')
 
@@ -180,6 +182,7 @@ def test_random_trend(close_figures):
     run_ucm('random_trend')
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 def test_cycle(close_figures):
     run_ucm('cycle')
 
@@ -196,10 +199,12 @@ def test_reg(close_figures):
     run_ucm('reg')
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 def test_rtrend_ar1(close_figures):
     run_ucm('rtrend_ar1')
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 @pytest.mark.slow
 def test_lltrend_cycle_seasonal_reg_ar1(close_figures):
     run_ucm('lltrend_cycle_seasonal_reg_ar1')
@@ -308,6 +313,7 @@ def test_forecast():
     assert_allclose(actual, desired)
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 def test_misc_exog():
     # Tests for missing data
     nobs = 20

diff --git a/statsmodels/tsa/statespace/tests/test_var.py b/statsmodels/tsa/statespace/tests/test_var.py
index 3029aa22c5627bef61a2bc0b8087ed2b1ef777c5..4003fbd2fd750f7f9eea50367c361b5d59f8c3ee 100644 (file)
--- a/statsmodels/tsa/statespace/tests/test_var.py
+++ b/statsmodels/tsa/statespace/tests/test_var.py
@@ -16,6 +16,8 @@ import os
 import numpy as np
 from numpy.testing import assert_allclose
 import pandas as pd
+from statsmodels.tsa.statespace.representation import warn_kalman
+import pytest
 
 from statsmodels.tsa.statespace import varmax
 from .results import results_var_R
@@ -101,6 +103,7 @@ def test_var_basic():
     # FEVD: TODO
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 def test_var_c():
     test = 'c'
 
@@ -121,6 +124,7 @@ def test_var_c():
     # FEVD: TODO
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 def test_var_ct():
     test = 'ct'
 
@@ -141,6 +145,7 @@ def test_var_ct():
     # FEVD: TODO
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 def test_var_ct_as_exog0():
     test = 'ct_as_exog0'
 
@@ -164,6 +169,7 @@ def test_var_ct_as_exog0():
     # FEVD: TODO
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 def test_var_ct_as_exog1():
     test = 'ct'
 
@@ -192,6 +198,7 @@ def test_var_ct_as_exog1():
     # FEVD: TODO
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 def test_var_ctt():
     test = 'ctt_as_exog1'
 
@@ -217,6 +224,7 @@ def test_var_ctt():
     # FEVD: TODO
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 def test_var_ct_exog():
     test = 'ct_exog'
 
@@ -241,6 +249,7 @@ def test_var_ct_exog():
     # FEVD: TODO
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 def test_var_c_2exog():
     test = 'c_2exog'
 

diff --git a/statsmodels/tsa/statespace/tests/test_varmax.py b/statsmodels/tsa/statespace/tests/test_varmax.py
index 7fc797e6f6065b310d8f92f2dc065617d774026c..bcb6267dabf1bef08612d8c2fa137ef4e1845acd 100644 (file)
--- a/statsmodels/tsa/statespace/tests/test_varmax.py
+++ b/statsmodels/tsa/statespace/tests/test_varmax.py
@@ -16,6 +16,7 @@ import pytest
 
 from statsmodels.tsa.statespace import varmax
 from statsmodels.iolib.summary import forg
+from statsmodels.tsa.statespace.representation import warn_kalman
 
 from .results import results_varmax
 
@@ -28,6 +29,7 @@ varmax_path = os.path.join('results', 'results_varmax_stata.csv')
 varmax_results = pd.read_csv(os.path.join(current_path, varmax_path))
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 class CheckVARMAX(object):
     """
     Test Vector Autoregression against Stata's `dfactor` code (Stata's
@@ -875,6 +877,7 @@ def test_misspecifications():
     warnings.resetwarnings()
 
 
+@pytest.mark.xfail(condition=bool(warn_kalman),reason='https://bugs.debian.org/924036')
 def test_misc_exog():
     # Tests for missing data
     nobs = 20