From 37b18a591484ffb16a00e0f9b997cf1eca189806 Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Sun, 11 Oct 2026 20:29:43 +0100 Subject: [PATCH 1/5] TST: Use eigh for the symmetric matrix in test_against_ols test_against_ols builds an orthogonal basis for the absorbed regressors from the eigenvectors of the symmetric matrix absorb.T @ absorb. np.linalg.eig is the solver for general matrices, and when the matrix is singular, which it is in 32 of the 72 configurations, it returns the whole array of eigenvectors as complex. Each of the 32 tests then emitted "ComplexWarning: Casting complex values to real discards the imaginary part" when IVData converted the regressors to float (iv/data.py:82). Use np.linalg.eigh. It is the solver for symmetric matrices and its eigenvectors are real. It orders the eigenvalues from the smallest, which the test already reverses with argsort(a)[::-1]. Only the span of the leading eigenvectors enters the OLS regression, so the test checks the same thing. Comparing the eig and eigh versions of every configuration: - The imaginary part of the eigenvectors that eig returned and the test used is exactly 0 in all 32 configurations. - The projection matrices on the two bases differ by at most 2e-16. - The OLS parameters, standard errors, covariance, residuals, R2 and s2 differ by at most 6e-15. Running test_against_ols with ComplexWarning turned into an error: 32 of 72 fail before this change, and all 72 pass after it. This is also needed before IVData rejects complex input instead of discarding the imaginary part, which would turn these warnings into errors. --- linearmodels/tests/iv/test_absorbing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/linearmodels/tests/iv/test_absorbing.py b/linearmodels/tests/iv/test_absorbing.py index ecb0203c05..2918c4074e 100644 --- a/linearmodels/tests/iv/test_absorbing.py +++ b/linearmodels/tests/iv/test_absorbing.py @@ -557,7 +557,8 @@ def test_against_ols(ols_data): absorb = (1.0 / root_w) * wabsorb rank = np.linalg.matrix_rank(absorb) if rank < absorb.shape[1]: - a, b = np.linalg.eig(absorb.T @ absorb) + # The matrix is symmetric: eigh has real eigenvectors, eig may not + a, b = np.linalg.eigh(absorb.T @ absorb) order = np.argsort(a)[::-1] a, b = a[order], b[:, order] z = absorb @ b From b84650a07cbc8f468a5c8cebf9eba37111f6d881 Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Sun, 11 Oct 2026 20:29:44 +0100 Subject: [PATCH 2/5] MAINT: Mark lsmr_options and the ordering module for removal in 8.0 Two deprecated features have no removal date. Say that they will be removed in linearmodels 8.0. AbsorbingLS.fit(lsmr_options=...) has been deprecated since 4.17 in favor of absorb_options. The FutureWarning and the ".. deprecated:: 4.17" note in the docstring now say that lsmr_options will be removed in linearmodels 8.0. The docstring also had a typo, "Options to ass to". linearmodels.__future__.ordering has been empty since 5.0, when the variable ordering that importing it enabled in 4.31 became the default. Importing it did nothing and said nothing. It now emits FutureWarning: linearmodels.__future__.ordering has no effect and will be removed in linearmodels 8.0 FutureWarning is shown by default. stacklevel=2 attributes the warning to the statement that imports the module, for `import linearmodels.__future__.ordering` and for `from linearmodels.__future__ import ordering`, because the warnings module skips the frames of the import machinery. Nothing in linearmodels or the examples imports the module. linearmodels/__future__/__init__.py is still empty, so importing the package does not warn. The new tests in tests/test_deprecations.py check that - both warnings have the text above and are attributed to the caller: the file that calls fit, and the file and line of the import statement; - the lsmr_options docstring says that it is deprecated and will be removed in 8.0; - lsmr_options is still passed on to the absorption, by comparing a fit with maxiter=1, which does not converge, to the same fit using absorb_options. The ordering module runs only the first time that it is imported, which may have been earlier in the test session. The tests remove it from sys.modules, run the import in a code object with a file name and a line number to see where the warning points, and restore the state afterwards. One more test imports it in a new interpreter, which is what a user sees. It takes a few seconds, because importing linearmodels does. Six of the seven new tests fail on main. The seventh checks that lsmr_options still works, which it also does on main. --- linearmodels/__future__/ordering.py | 16 +++ linearmodels/iv/absorbing.py | 8 +- linearmodels/tests/test_deprecations.py | 129 ++++++++++++++++++++++++ 3 files changed, 150 insertions(+), 3 deletions(-) create mode 100644 linearmodels/tests/test_deprecations.py diff --git a/linearmodels/__future__/ordering.py b/linearmodels/__future__/ordering.py index e69de29bb2..58f050489f 100644 --- a/linearmodels/__future__/ordering.py +++ b/linearmodels/__future__/ordering.py @@ -0,0 +1,16 @@ +""" +Opt in to formulas that do not sort variables. + +This was the future behavior in linearmodels 4.31. It has been the default, and +importing this module has had no effect, since linearmodels 5.0. The module is +kept so that existing code keeps working and will be removed in linearmodels 8.0. +""" + +import warnings + +warnings.warn( + "linearmodels.__future__.ordering has no effect and will be removed in " + "linearmodels 8.0", + FutureWarning, + stacklevel=2, +) diff --git a/linearmodels/iv/absorbing.py b/linearmodels/iv/absorbing.py index 9f001ea952..b14fcf74c0 100644 --- a/linearmodels/iv/absorbing.py +++ b/linearmodels/iv/absorbing.py @@ -1047,11 +1047,12 @@ def fit( re-fitting large models when the set of absorbed variables and interactions are identical. lsmr_options : dict - Options to ass to scipy.sparse.linalg.lsmr. + Options to pass to scipy.sparse.linalg.lsmr. .. deprecated:: 4.17 - Use absorb_options to pass options + Use absorb_options to pass options. ``lsmr_options`` will be + removed in linearmodels 8.0. cov_config Additional parameters to pass to covariance estimator. The list @@ -1089,7 +1090,8 @@ def fit( if absorb_options is not None: raise ValueError("absorb_options cannot be used with lsmr_options") warnings.warn( - "lsmr_options is deprecated. Use absorb_options.", + "lsmr_options is deprecated and will be removed in linearmodels " + "8.0. Use absorb_options.", FutureWarning, stacklevel=2, ) diff --git a/linearmodels/tests/test_deprecations.py b/linearmodels/tests/test_deprecations.py new file mode 100644 index 0000000000..7155d0618b --- /dev/null +++ b/linearmodels/tests/test_deprecations.py @@ -0,0 +1,129 @@ +""" +Tests of the features that are deprecated and will be removed in linearmodels 8.0 + +Remove the tests of a feature when the feature is removed. +""" + +import importlib +from pathlib import Path +import re +import subprocess +import sys + +import numpy as np +from numpy.testing import assert_allclose +import pandas as pd +import pytest + +from linearmodels.iv.absorbing import AbsorbingLS + +LSMR_OPTIONS_MESSAGE = ( + "lsmr_options is deprecated and will be removed in linearmodels 8.0" +) +ORDERING = "linearmodels.__future__.ordering" +ORDERING_MESSAGE = ( + "linearmodels.__future__.ordering has no effect and will be removed in " + "linearmodels 8.0" +) + + +@pytest.fixture +def absorbing_data(): + # Two crossed factors, so that LSMR needs more than one iteration + rng = np.random.default_rng(20261011) + nobs = 300 + cat = pd.DataFrame( + { + "a": pd.Series(rng.integers(0, 12, nobs), dtype="category"), + "b": pd.Series(rng.integers(0, 9, nobs), dtype="category"), + } + ) + x = pd.DataFrame(rng.standard_normal((nobs, 2)), columns=["x0", "x1"]) + effects = cat["a"].cat.codes.to_numpy() + 0.5 * cat["b"].cat.codes.to_numpy() + y = pd.Series(x.sum(axis=1).to_numpy() + effects + rng.standard_normal(nobs)) + return y, x, cat + + +def test_lsmr_options_warns_of_removal(absorbing_data): + y, x, cat = absorbing_data + mod = AbsorbingLS(y, x, absorb=cat) + with pytest.warns(FutureWarning, match=re.escape(LSMR_OPTIONS_MESSAGE)) as record: + mod.fit(method="lsmr", lsmr_options={}) + matches = [w for w in record if LSMR_OPTIONS_MESSAGE in str(w.message)] + assert len(matches) == 1 + # The warning points at the call to fit + assert Path(matches[0].filename).resolve() == Path(__file__).resolve() + + +def test_lsmr_options_docstring_says_when_it_is_removed(): + doc = AbsorbingLS.fit.__doc__ + if doc is None: + pytest.skip("Docstrings are not available") + # The text of the parameter, with whitespace normalized to ignore line breaks + text = doc.partition("lsmr_options :")[2].partition("cov_config")[0] + block = " ".join(text.split()) + assert ".. deprecated:: 4.17" in block + assert "will be removed in linearmodels 8.0" in block + + +def test_lsmr_options_are_still_used(absorbing_data): + y, x, cat = absorbing_data + # One iteration is far from the solution that the default options find + options = {"maxiter": 1} + default = AbsorbingLS(y, x, absorb=cat).fit(method="lsmr", use_cache=False) + expected = AbsorbingLS(y, x, absorb=cat).fit( + method="lsmr", absorb_options=options, use_cache=False + ) + assert np.max(np.abs(default.params - expected.params)) > 1e-3 + + with pytest.warns(FutureWarning, match="lsmr_options"): + res = AbsorbingLS(y, x, absorb=cat).fit( + method="lsmr", lsmr_options=options, use_cache=False + ) + assert_allclose(res.params, expected.params, rtol=1e-10) + assert_allclose(res.std_errors, expected.std_errors, rtol=1e-10) + + +@pytest.fixture +def forget_ordering(): + """Make the next import of the ordering module run the module again""" + package = importlib.import_module("linearmodels.__future__") + module = sys.modules.pop(ORDERING, None) + attribute = package.__dict__.pop("ordering", None) + yield + sys.modules.pop(ORDERING, None) + package.__dict__.pop("ordering", None) + if module is not None: + sys.modules[ORDERING] = module + if attribute is not None: + package.ordering = attribute + + +@pytest.mark.parametrize( + "statement", + [ + "import linearmodels.__future__.ordering", + "import linearmodels.__future__.ordering as ordering", + "from linearmodels.__future__ import ordering", + ], +) +def test_ordering_warns_of_removal(forget_ordering, statement): + # The module only runs on its first import, so run the statement as if it was + # in another file, on line 2, to check where the warning points + code = compile(f"\n{statement}\n", "importing_module.py", "exec") + with pytest.warns(FutureWarning, match=re.escape(ORDERING_MESSAGE)) as record: + exec(code, {}) # noqa: S102 + matches = [w for w in record if ORDERING_MESSAGE in str(w.message)] + assert len(matches) == 1 + assert matches[0].filename == "importing_module.py" + assert matches[0].lineno == 2 + + +def test_ordering_warns_in_a_new_interpreter(): + # The module only runs on its first import in a process, so use a new interpreter + # to check what a user sees. -W makes this independent of PYTHONWARNINGS + code = "\nfrom linearmodels.__future__ import ordering\n" + cmd = [sys.executable, "-W", "default::FutureWarning", "-c", code] + proc = subprocess.run(cmd, capture_output=True, text=True, timeout=300, check=False) + assert proc.returncode == 0, proc.stderr + assert f":2: FutureWarning: {ORDERING_MESSAGE}" in proc.stderr From b251b20c8c1cae11ef618a00f55ab8f1e5d9a7f8 Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Sun, 11 Oct 2026 20:29:44 +0100 Subject: [PATCH 3/5] TST: Do not select the Windows event loop policy on Python 3.14+ test_examples.py switches asyncio to the Windows selector event loop policy so that zmq, which the notebook kernel client uses, does not warn about the default Proactor loop. WindowsSelectorEventLoopPolicy, get_event_loop_policy and set_event_loop_policy are deprecated in Python 3.14 and will be removed in 3.16. Every xdist worker emitted four DeprecationWarnings when it imported the test module, 16 in a run with -n 4. Only select the policy before Python 3.14. The (3, 8) lower bound is below the minimum Python version, so drop it. I checked that the notebooks run without the policy on Windows 11 with Python 3.14.6, pyzmq 27.1.0, jupyter_client 8.9.1, ipykernel 7.3.0, nbclient 0.11.0 and tornado 6.5.7, using python -m pytest linearmodels/tests/test_examples.py -n 4: | Case | Result | |---|---| | Policy selected (before this change) | 12 passed, 16 warnings (DeprecationWarning) | | Policy not selected | 12 passed, 4 warnings (pyzmq RuntimeWarning) | | Policy not selected, pyzmq warning ignored | 12 passed, 0 warnings | Without the policy pyzmq warns once per worker that the Proactor loop has no add_reader, that it registers a selector thread instead, and recommends the deprecated function. It only describes how zmq works on this loop, so ignore it in test_notebook. --- linearmodels/tests/test_examples.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/linearmodels/tests/test_examples.py b/linearmodels/tests/test_examples.py index ca2f3ad3c1..23b63eb929 100644 --- a/linearmodels/tests/test_examples.py +++ b/linearmodels/tests/test_examples.py @@ -21,7 +21,10 @@ kernels = jupyter_client.kernelspec.find_kernel_specs() plat_win = sys.platform.startswith("win") - if plat_win and (3, 8) <= sys.version_info < (3, 15): # pragma: no cover + # The default Proactor event loop on Windows lacks the add_reader that zmq needs, + # and zmq warns that it falls back to a selector thread. The policy functions + # that avoid this are deprecated in Python 3.14, so only use them before that. + if plat_win and sys.version_info < (3, 14): # pragma: no cover import asyncio try: @@ -55,6 +58,10 @@ def notebook(request): return request.param +# Windows, Python 3.14+: ignore the zmq warning about the Proactor loop (see above) +@pytest.mark.filterwarnings( + "ignore:Proactor event loop does not implement add_reader:RuntimeWarning" +) @pytest.mark.slow @pytest.mark.example @pytest.mark.skipif(SKIP, reason="Required packages not available") From 6507e533acb6d2e056964477e898c74dd1ba5078 Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Sun, 11 Oct 2026 20:29:44 +0100 Subject: [PATCH 4/5] TST: Compare C statistic p-values to R with a relative tolerance only test_c_stat_against_r compared p-values with rtol=1e-5, atol=1e-12 and a comment that the p-value is 1 - cdf, so it cannot resolve probabilities below about 1e-16. That stopped being true in #742: p-values are survival functions, which are accurate in the far tail. The absolute tolerance only hides errors in small p-values. Four of the 23 reference p-values are below 1e-12 (the smallest is 2e-29), so any computed value between 0 and 1e-12 passed for them. Use rtol=1e-8 and no atol, and replace the stale comment. The largest relative difference between a p-value and the R reference in the table is 7e-13, so the tolerance leaves a margin of four orders of magnitude. --- linearmodels/tests/iv/test_c_stat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/linearmodels/tests/iv/test_c_stat.py b/linearmodels/tests/iv/test_c_stat.py index cc47bf3556..478aec7264 100644 --- a/linearmodels/tests/iv/test_c_stat.py +++ b/linearmodels/tests/iv/test_c_stat.py @@ -75,8 +75,8 @@ def test_c_stat_against_r(row): res = build_model(row).fit(cov_type="robust") c_stat = res.c_stat(row.tested.split() or None) assert_allclose(c_stat.stat, row.c_stat, rtol=1e-6) - # pval is 1 - cdf, so it cannot resolve tail probabilities below about 1e-16 - assert_allclose(c_stat.pval, row.pvalue, rtol=1e-5, atol=1e-12) + # pval is the survival function, so it is accurate far into the tail: no atol + assert_allclose(c_stat.pval, row.pvalue, rtol=1e-8) assert c_stat.df == row.df # The Hansen J statistic of the original model uses the same GMM steps assert_allclose(res.j_stat.stat, row.hansen_j, rtol=1e-6, atol=1e-8) From c8a518f244bb83a69b76fa4832f53ba0efc8191c Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Sun, 11 Oct 2026 20:29:45 +0100 Subject: [PATCH 5/5] BUG: Do not modify the absorb_options passed to AbsorbingLS.fit The HDFE path sets absorb_options["drop_singletons"] = False before it calls pyhdfe.create. It did this on the dict that the caller passed, so after fit the caller's dict had a new key. That key is not an option of LSMR, so reusing the dict for a model that is fitted with method="lsmr" failed: options = {} AbsorbingLS(y, x, absorb=cat).fit(absorb_options=options) # options is now {"drop_singletons": False} AbsorbingLS(y, x, absorb=cat).fit(absorb_options=options, method="lsmr") # TypeError: lsmr() got an unexpected keyword argument 'drop_singletons' Copy the dict in _first_time_fit. The deprecated lsmr_options were already copied. The new test checks both symptoms. On main it fails on the first assertion, because the options that were passed in have gained a key, and the LSMR fit that follows raises the TypeError above. The LSMR fit does not use the cache, which could otherwise hold the result of an earlier fit and skip the call that fails. This is a different issue from the lsmr_options deprecation and is in its own commit. --- linearmodels/iv/absorbing.py | 2 +- linearmodels/tests/iv/test_absorbing.py | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/linearmodels/iv/absorbing.py b/linearmodels/iv/absorbing.py index b14fcf74c0..7f2e23f8c9 100644 --- a/linearmodels/iv/absorbing.py +++ b/linearmodels/iv/absorbing.py @@ -922,7 +922,7 @@ def _first_time_fit( mu_dep = (root_w.T @ dep) / denom mu_exog = (root_w.T @ exog) / denom - absorb_options = {} if absorb_options is None else absorb_options + absorb_options = {} if absorb_options is None else dict(absorb_options) assert isinstance(self._regressors, csc_array) if self._regressors.shape[1] > 0: if use_hdfe: diff --git a/linearmodels/tests/iv/test_absorbing.py b/linearmodels/tests/iv/test_absorbing.py index 2918c4074e..2e531a8373 100644 --- a/linearmodels/tests/iv/test_absorbing.py +++ b/linearmodels/tests/iv/test_absorbing.py @@ -757,6 +757,29 @@ def test_options(random_gen): mod.fit(absorb_options={"atol": 1e-7, "btol": 1e-7}, method="hdfe") +def test_absorb_options_are_not_modified(random_gen): + absorb = random_gen.randint(0, 10, size=1000) + x = random_gen.standard_normal((1000, 3)) + y = random_gen.standard_normal(1000) + dfd = {f"x{i}": pd.Series(x[:, i]) for i in range(3)} + dfd.update({"c": pd.Series(absorb, dtype="category"), "y": pd.Series(y)}) + df = pd.DataFrame(dfd) + + y = df.y + x = df.iloc[:, :3] + # HDFE sets drop_singletons for pyhdfe, which must not change the caller's dict + options = {"compute_degrees": False} + AbsorbingLS(y, x, absorb=df[["c"]]).fit(absorb_options=options) + assert options == {"compute_degrees": False} + + # Otherwise these options could not be used by LSMR, which rejects drop_singletons + options = {} + AbsorbingLS(y, x, absorb=df[["c"]]).fit(absorb_options=options) + mod = AbsorbingLS(y, x, absorb=df[["c"]]) + mod.fit(absorb_options=options, method="lsmr", use_cache=False) + assert options == {} + + def test_lsmr_annihilate_empty(): gen = np.random.default_rng(0) x = csc_array(gen.standard_normal((1000, 2)))