diff --git a/linearmodels/__future__/ordering.py b/linearmodels/__future__/ordering.py index e69de29bb2..58f050489f 100644 --- a/linearmodels/__future__/ordering.py +++ b/linearmodels/__future__/ordering.py @@ -0,0 +1,16 @@ +""" +Opt in to formulas that do not sort variables. + +This was the future behavior in linearmodels 4.31. It has been the default, and +importing this module has had no effect, since linearmodels 5.0. The module is +kept so that existing code keeps working and will be removed in linearmodels 8.0. +""" + +import warnings + +warnings.warn( + "linearmodels.__future__.ordering has no effect and will be removed in " + "linearmodels 8.0", + FutureWarning, + stacklevel=2, +) diff --git a/linearmodels/iv/absorbing.py b/linearmodels/iv/absorbing.py index 9f001ea952..7f2e23f8c9 100644 --- a/linearmodels/iv/absorbing.py +++ b/linearmodels/iv/absorbing.py @@ -922,7 +922,7 @@ def _first_time_fit( mu_dep = (root_w.T @ dep) / denom mu_exog = (root_w.T @ exog) / denom - absorb_options = {} if absorb_options is None else absorb_options + absorb_options = {} if absorb_options is None else dict(absorb_options) assert isinstance(self._regressors, csc_array) if self._regressors.shape[1] > 0: if use_hdfe: @@ -1047,11 +1047,12 @@ def fit( re-fitting large models when the set of absorbed variables and interactions are identical. lsmr_options : dict - Options to ass to scipy.sparse.linalg.lsmr. + Options to pass to scipy.sparse.linalg.lsmr. .. deprecated:: 4.17 - Use absorb_options to pass options + Use absorb_options to pass options. ``lsmr_options`` will be + removed in linearmodels 8.0. cov_config Additional parameters to pass to covariance estimator. The list @@ -1089,7 +1090,8 @@ def fit( if absorb_options is not None: raise ValueError("absorb_options cannot be used with lsmr_options") warnings.warn( - "lsmr_options is deprecated. Use absorb_options.", + "lsmr_options is deprecated and will be removed in linearmodels " + "8.0. Use absorb_options.", FutureWarning, stacklevel=2, ) diff --git a/linearmodels/tests/iv/test_absorbing.py b/linearmodels/tests/iv/test_absorbing.py index ecb0203c05..2e531a8373 100644 --- a/linearmodels/tests/iv/test_absorbing.py +++ b/linearmodels/tests/iv/test_absorbing.py @@ -557,7 +557,8 @@ def test_against_ols(ols_data): absorb = (1.0 / root_w) * wabsorb rank = np.linalg.matrix_rank(absorb) if rank < absorb.shape[1]: - a, b = np.linalg.eig(absorb.T @ absorb) + # The matrix is symmetric: eigh has real eigenvectors, eig may not + a, b = np.linalg.eigh(absorb.T @ absorb) order = np.argsort(a)[::-1] a, b = a[order], b[:, order] z = absorb @ b @@ -756,6 +757,29 @@ def test_options(random_gen): mod.fit(absorb_options={"atol": 1e-7, "btol": 1e-7}, method="hdfe") +def test_absorb_options_are_not_modified(random_gen): + absorb = random_gen.randint(0, 10, size=1000) + x = random_gen.standard_normal((1000, 3)) + y = random_gen.standard_normal(1000) + dfd = {f"x{i}": pd.Series(x[:, i]) for i in range(3)} + dfd.update({"c": pd.Series(absorb, dtype="category"), "y": pd.Series(y)}) + df = pd.DataFrame(dfd) + + y = df.y + x = df.iloc[:, :3] + # HDFE sets drop_singletons for pyhdfe, which must not change the caller's dict + options = {"compute_degrees": False} + AbsorbingLS(y, x, absorb=df[["c"]]).fit(absorb_options=options) + assert options == {"compute_degrees": False} + + # Otherwise these options could not be used by LSMR, which rejects drop_singletons + options = {} + AbsorbingLS(y, x, absorb=df[["c"]]).fit(absorb_options=options) + mod = AbsorbingLS(y, x, absorb=df[["c"]]) + mod.fit(absorb_options=options, method="lsmr", use_cache=False) + assert options == {} + + def test_lsmr_annihilate_empty(): gen = np.random.default_rng(0) x = csc_array(gen.standard_normal((1000, 2))) diff --git a/linearmodels/tests/iv/test_c_stat.py b/linearmodels/tests/iv/test_c_stat.py index cc47bf3556..478aec7264 100644 --- a/linearmodels/tests/iv/test_c_stat.py +++ b/linearmodels/tests/iv/test_c_stat.py @@ -75,8 +75,8 @@ def test_c_stat_against_r(row): res = build_model(row).fit(cov_type="robust") c_stat = res.c_stat(row.tested.split() or None) assert_allclose(c_stat.stat, row.c_stat, rtol=1e-6) - # pval is 1 - cdf, so it cannot resolve tail probabilities below about 1e-16 - assert_allclose(c_stat.pval, row.pvalue, rtol=1e-5, atol=1e-12) + # pval is the survival function, so it is accurate far into the tail: no atol + assert_allclose(c_stat.pval, row.pvalue, rtol=1e-8) assert c_stat.df == row.df # The Hansen J statistic of the original model uses the same GMM steps assert_allclose(res.j_stat.stat, row.hansen_j, rtol=1e-6, atol=1e-8) diff --git a/linearmodels/tests/test_deprecations.py b/linearmodels/tests/test_deprecations.py new file mode 100644 index 0000000000..7155d0618b --- /dev/null +++ b/linearmodels/tests/test_deprecations.py @@ -0,0 +1,129 @@ +""" +Tests of the features that are deprecated and will be removed in linearmodels 8.0 + +Remove the tests of a feature when the feature is removed. +""" + +import importlib +from pathlib import Path +import re +import subprocess +import sys + +import numpy as np +from numpy.testing import assert_allclose +import pandas as pd +import pytest + +from linearmodels.iv.absorbing import AbsorbingLS + +LSMR_OPTIONS_MESSAGE = ( + "lsmr_options is deprecated and will be removed in linearmodels 8.0" +) +ORDERING = "linearmodels.__future__.ordering" +ORDERING_MESSAGE = ( + "linearmodels.__future__.ordering has no effect and will be removed in " + "linearmodels 8.0" +) + + +@pytest.fixture +def absorbing_data(): + # Two crossed factors, so that LSMR needs more than one iteration + rng = np.random.default_rng(20261011) + nobs = 300 + cat = pd.DataFrame( + { + "a": pd.Series(rng.integers(0, 12, nobs), dtype="category"), + "b": pd.Series(rng.integers(0, 9, nobs), dtype="category"), + } + ) + x = pd.DataFrame(rng.standard_normal((nobs, 2)), columns=["x0", "x1"]) + effects = cat["a"].cat.codes.to_numpy() + 0.5 * cat["b"].cat.codes.to_numpy() + y = pd.Series(x.sum(axis=1).to_numpy() + effects + rng.standard_normal(nobs)) + return y, x, cat + + +def test_lsmr_options_warns_of_removal(absorbing_data): + y, x, cat = absorbing_data + mod = AbsorbingLS(y, x, absorb=cat) + with pytest.warns(FutureWarning, match=re.escape(LSMR_OPTIONS_MESSAGE)) as record: + mod.fit(method="lsmr", lsmr_options={}) + matches = [w for w in record if LSMR_OPTIONS_MESSAGE in str(w.message)] + assert len(matches) == 1 + # The warning points at the call to fit + assert Path(matches[0].filename).resolve() == Path(__file__).resolve() + + +def test_lsmr_options_docstring_says_when_it_is_removed(): + doc = AbsorbingLS.fit.__doc__ + if doc is None: + pytest.skip("Docstrings are not available") + # The text of the parameter, with whitespace normalized to ignore line breaks + text = doc.partition("lsmr_options :")[2].partition("cov_config")[0] + block = " ".join(text.split()) + assert ".. deprecated:: 4.17" in block + assert "will be removed in linearmodels 8.0" in block + + +def test_lsmr_options_are_still_used(absorbing_data): + y, x, cat = absorbing_data + # One iteration is far from the solution that the default options find + options = {"maxiter": 1} + default = AbsorbingLS(y, x, absorb=cat).fit(method="lsmr", use_cache=False) + expected = AbsorbingLS(y, x, absorb=cat).fit( + method="lsmr", absorb_options=options, use_cache=False + ) + assert np.max(np.abs(default.params - expected.params)) > 1e-3 + + with pytest.warns(FutureWarning, match="lsmr_options"): + res = AbsorbingLS(y, x, absorb=cat).fit( + method="lsmr", lsmr_options=options, use_cache=False + ) + assert_allclose(res.params, expected.params, rtol=1e-10) + assert_allclose(res.std_errors, expected.std_errors, rtol=1e-10) + + +@pytest.fixture +def forget_ordering(): + """Make the next import of the ordering module run the module again""" + package = importlib.import_module("linearmodels.__future__") + module = sys.modules.pop(ORDERING, None) + attribute = package.__dict__.pop("ordering", None) + yield + sys.modules.pop(ORDERING, None) + package.__dict__.pop("ordering", None) + if module is not None: + sys.modules[ORDERING] = module + if attribute is not None: + package.ordering = attribute + + +@pytest.mark.parametrize( + "statement", + [ + "import linearmodels.__future__.ordering", + "import linearmodels.__future__.ordering as ordering", + "from linearmodels.__future__ import ordering", + ], +) +def test_ordering_warns_of_removal(forget_ordering, statement): + # The module only runs on its first import, so run the statement as if it was + # in another file, on line 2, to check where the warning points + code = compile(f"\n{statement}\n", "importing_module.py", "exec") + with pytest.warns(FutureWarning, match=re.escape(ORDERING_MESSAGE)) as record: + exec(code, {}) # noqa: S102 + matches = [w for w in record if ORDERING_MESSAGE in str(w.message)] + assert len(matches) == 1 + assert matches[0].filename == "importing_module.py" + assert matches[0].lineno == 2 + + +def test_ordering_warns_in_a_new_interpreter(): + # The module only runs on its first import in a process, so use a new interpreter + # to check what a user sees. -W makes this independent of PYTHONWARNINGS + code = "\nfrom linearmodels.__future__ import ordering\n" + cmd = [sys.executable, "-W", "default::FutureWarning", "-c", code] + proc = subprocess.run(cmd, capture_output=True, text=True, timeout=300, check=False) + assert proc.returncode == 0, proc.stderr + assert f":2: FutureWarning: {ORDERING_MESSAGE}" in proc.stderr diff --git a/linearmodels/tests/test_examples.py b/linearmodels/tests/test_examples.py index ca2f3ad3c1..23b63eb929 100644 --- a/linearmodels/tests/test_examples.py +++ b/linearmodels/tests/test_examples.py @@ -21,7 +21,10 @@ kernels = jupyter_client.kernelspec.find_kernel_specs() plat_win = sys.platform.startswith("win") - if plat_win and (3, 8) <= sys.version_info < (3, 15): # pragma: no cover + # The default Proactor event loop on Windows lacks the add_reader that zmq needs, + # and zmq warns that it falls back to a selector thread. The policy functions + # that avoid this are deprecated in Python 3.14, so only use them before that. + if plat_win and sys.version_info < (3, 14): # pragma: no cover import asyncio try: @@ -55,6 +58,10 @@ def notebook(request): return request.param +# Windows, Python 3.14+: ignore the zmq warning about the Proactor loop (see above) +@pytest.mark.filterwarnings( + "ignore:Proactor event loop does not implement add_reader:RuntimeWarning" +) @pytest.mark.slow @pytest.mark.example @pytest.mark.skipif(SKIP, reason="Required packages not available")