From d90294301077970a25211bd201bdd455521e8385 Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Sun, 11 Oct 2026 22:31:08 +0100 Subject: [PATCH] BUG: Raise the missing index error from every group operation of PanelData #716 made PanelData.entity_ids and PanelData.time_ids raise a ValueError that says that the entity or time index contains missing values (NaN). The estimators that do not use those properties still failed with errors that do not say what is wrong, or did not fail. Reproduction (reg1/repro_leads.py): a panel with 20 entities and 5 periods, and a NaN in the entity label of one row. model main now PooledOLS, FirstDifferenceOLS ValueError (#716) same PanelOLS(time_effects=True) ValueError (#716) same PanelOLS(entity_effects=True) LinAlgError: SVD did not ValueError (#716) converge PanelOLS(entity and time effects) LinAlgError: SVD did not ValueError (#716) converge in Linear Least Squares RandomEffects ValueError: array must not ValueError (#716) contain infs or NaNs A NaN (or NaT) in the time label is covered for every model except PanelOLS(time_effects=True), which raised LinAlgError: SVD did not converge. pandas encodes a missing label as -1 in the codes of the level. The effects demean by the groups of the index, which are found with groupby, and a groupby drops the rows with a missing label. The demeaned data have NaN in those rows, and the check of the rank of the regressors fails in the SVD. Other group operations of PanelData do not fail and are wrong. dummies uses the last label of the level for a row with a missing label, so that the row is in the last entity, and the entities and time periods of the 3-d view (values3d, and so first_difference) have the same problem. The fix is a method, PanelData._check_index, with the message of #716. It is used by entity_ids and time_ids, as before, and now also by the operations that group by the index: * demean, for the level of its group, and both for "both", * mean, count and dummies, for the level of their group, * the panel view that values3d and first_difference use, for both levels. The levels that an operation does not use are not checked. An index with a missing label can still be used to construct a PanelData, as #716 tests. I ran 52 combinations of the estimators with options (cov_type, clusters, weights, low_memory, use_lsdv, auto_df, count_effects, drop_absorbed, check_rank, other_effects) with a missing label in the entity index and in the time index. 28 of the 104 fits did not raise the ValueError on main, 26 with LinAlgError and 2 with RandomEffects' "array must not contain infs or NaNs". All of them do now. The estimates and the results of panels without a missing label are not changed: the panel tests pass (3,398 passed, including the new tests, 61 skipped and 47 xfailed). The change is in panel/data.py only, so it does not touch the code that the input alignment branch changes in panel/model.py. The new tests are in test_missing_index.py. They fit each estimator with the label of the entity and of the time missing, using integer labels and string and date labels, with weights, covariance estimators, low memory demeaning and formulas, and call each operation of PanelData. 52 of the 158 tests fail without the change. --- linearmodels/panel/data.py | 51 ++-- .../tests/panel/test_missing_index.py | 228 ++++++++++++++++++ 2 files changed, 265 insertions(+), 14 deletions(-) create mode 100644 linearmodels/tests/panel/test_missing_index.py diff --git a/linearmodels/panel/data.py b/linearmodels/panel/data.py index f2c5b5ce2a..aecca87421 100644 --- a/linearmodels/panel/data.py +++ b/linearmodels/panel/data.py @@ -158,6 +158,10 @@ class PanelData: Index level 1 is time. The columns are the variables. MultiIndex Series are also accepted and treated as single column MultiIndex DataFrames. + Missing values (NaN) in the entity or time index are not supported. The + methods that group by the index raise a ValueError if the level that they + use has a missing label. + Raises ------ TypeError @@ -270,6 +274,7 @@ def __init__( def panel(self) -> _Panel: """pandas Panel view of data""" if self._panel is None: + self._check_index(0, 1) self._panel = _Panel(self._frame) assert self._panel is not None return self._panel @@ -360,6 +365,30 @@ def entities(self) -> list[linearmodels.typing.Label]: index = self.index return list(index.levels[0][index.codes[0]].unique()) + def _check_index(self, *levels: int) -> None: + """ + Raise a ValueError if a level of the index has missing values (NaN) + + Parameters + ---------- + *levels : int + Levels to check, 0 for the entity and 1 for the time + + Notes + ----- + pandas encodes a missing label as -1 in the codes of the level. The + operations that group by a level have to use this to reject it since + a groupby drops the rows that have a missing label, and the codes + select the last label. + """ + for level in levels: + if np.any(self.index.codes[level] < 0): + name = "entity" if level == 0 else "time" + raise ValueError( + f"The {name} index contains missing values (NaN), which are not " + f"supported and are encoded as -1 in the {name} ids." + ) + @property def entity_ids(self) -> linearmodels.typing.data.AnyIntArray: """ @@ -370,13 +399,8 @@ def entity_ids(self) -> linearmodels.typing.data.AnyIntArray: ndarray 2d array containing entity ids corresponding dataframe view """ - index = self.index - if np.any(index.codes[0] < 0): - raise ValueError( - "The entity index contains missing values (NaN), which are not " - "supported and are encoded as -1 in the entity ids." - ) - return np.asarray(index.codes[0])[:, None] + self._check_index(0) + return np.asarray(self.index.codes[0])[:, None] @property def time_ids(self) -> linearmodels.typing.data.AnyIntArray: @@ -388,13 +412,8 @@ def time_ids(self) -> linearmodels.typing.data.AnyIntArray: ndarray 2d array containing time ids corresponding dataframe view """ - index = self.index - if np.any(index.codes[1] < 0): - raise ValueError( - "The time index contains missing values (NaN), which are not " - "supported and are encoded as -1 in the time ids." - ) - return np.asarray(index.codes[1])[:, None] + self._check_index(1) + return np.asarray(self.index.codes[1])[:, None] def _demean_both_low_mem(self, weights: PanelData | None) -> PanelData: groups = PanelData( @@ -597,6 +616,7 @@ def demean( """ if group not in ("entity", "time", "both"): raise ValueError("Group must be one of 'entity', 'time' or 'both'") + self._check_index(*{"entity": (0,), "time": (1,), "both": (0, 1)}[group]) if group == "both": if not low_memory: return self._demean_both(weights) @@ -654,6 +674,7 @@ def count(self, group: str = "entity") -> DataFrame: Counts according to type. Either (entity by var) or (time by var) """ level = 0 if group == "entity" else 1 + self._check_index(level) reindex = self.entities if group == "entity" else self.time out = self._frame.groupby(level=level).count() @@ -694,6 +715,7 @@ def mean( Data mean according to type. Either (entity by var) or (time by var) """ level = 0 if group == "entity" else 1 + self._check_index(level) if weights is None: mu = self._frame.groupby(level=level).mean() else: @@ -764,6 +786,7 @@ def dummies(self, group: str = "entity", drop_first: bool = False) -> DataFrame: if group not in ("entity", "time"): raise ValueError("Group must be one of 'entity' or 'time'") axis = 0 if group == "entity" else 1 + self._check_index(axis) labels = self.index.codes levels = self.index.levels cat = Categorical(levels[axis][labels[axis]]) diff --git a/linearmodels/tests/panel/test_missing_index.py b/linearmodels/tests/panel/test_missing_index.py new file mode 100644 index 0000000000..11465a513c --- /dev/null +++ b/linearmodels/tests/panel/test_missing_index.py @@ -0,0 +1,228 @@ +""" +Missing values (NaN) in the entity or time index of a panel + +pandas encodes a missing label in a level of an index as -1 in the codes of the +level. Operations that group by the index, such as demeaning by entity, silently +drop the rows with a missing label, and the data that they return contain NaN. An +estimator that uses them fails later with an error from the linear algebra that +does not say what is wrong, e.g., "SVD did not converge". + +Every estimator has to raise the ValueError that says that the entity or the time +index contains missing values. PanelData.entity_ids and PanelData.time_ids do, but +the other operations that use the index did not, and so PanelOLS with entity +effects, PanelOLS with time effects and RandomEffects did not. +""" + +from linearmodels.compat.pandas import ANNUAL_FREQ + +import numpy as np +import pandas as pd +import pytest + +from linearmodels.panel.data import PanelData +from linearmodels.panel.model import ( + BetweenOLS, + FamaMacBeth, + FirstDifferenceOLS, + PanelOLS, + PooledOLS, + RandomEffects, +) + +NENTITY = 20 +NTIME = 5 +MESSAGES = { + "entity": "The entity index contains missing values", + "time": "The time index contains missing values", +} + + +def make_panel(level, labels): + """ + Panel where the label of the entity or the time of one row is missing + + The labels are integers, or strings for the entities and dates for the time. + The panel has no missing labels if level is None. + """ + rng = np.random.default_rng(0) + if labels == "int": + entities = np.repeat(np.arange(NENTITY, dtype=float), NTIME) + times = np.tile(np.arange(NTIME, dtype=float), NENTITY) + missing = np.nan + else: + names = np.array([f"firm{i}" for i in range(NENTITY)], dtype=object) + entities = np.repeat(names, NTIME) + dates = pd.date_range("2001-12-31", periods=NTIME, freq=ANNUAL_FREQ) + times = np.tile(dates.to_numpy(), NENTITY) + missing = None if level == "entity" else np.datetime64("NaT", "ns") + labels_by_level = {"entity": entities, "time": times} + if level is not None: + labels_by_level[level][5] = missing + index = pd.MultiIndex.from_arrays( + [labels_by_level["entity"], labels_by_level["time"]], names=["firm", "year"] + ) + nobs = NENTITY * NTIME + data = pd.DataFrame( + { + "y": rng.standard_normal(nobs), + "x": rng.standard_normal(nobs), + "const": 1.0, + "w": rng.chisquare(5, nobs) + 0.5, + "group": np.arange(nobs) % 4, + }, + index=index, + ) + ncodes = { + name: int(np.sum(data.index.codes[i] == -1)) + for i, name in enumerate(["entity", "time"]) + } + assert ncodes == {"entity": int(level == "entity"), "time": int(level == "time")} + return data + + +MODELS = { + "pooled": lambda d: PooledOLS(d.y, d[["const", "x"]]), + "pooled-weights": lambda d: PooledOLS(d.y, d[["const", "x"]], weights=d.w), + "between": lambda d: BetweenOLS(d.y, d[["const", "x"]]), + "fama-macbeth": lambda d: FamaMacBeth(d.y, d[["const", "x"]]), + "random-effects": lambda d: RandomEffects(d.y, d[["const", "x"]]), + "random-effects-weights": lambda d: RandomEffects( + d.y, d[["const", "x"]], weights=d.w + ), + "first-difference": lambda d: FirstDifferenceOLS(d.y, d[["x"]]), + "panel-no-effects": lambda d: PanelOLS(d.y, d[["const", "x"]]), + "panel-entity": lambda d: PanelOLS(d.y, d[["x"]], entity_effects=True), + "panel-entity-weights": lambda d: PanelOLS( + d.y, d[["x"]], entity_effects=True, weights=d.w + ), + "panel-time": lambda d: PanelOLS(d.y, d[["x"]], time_effects=True), + "panel-time-weights": lambda d: PanelOLS( + d.y, d[["x"]], time_effects=True, weights=d.w + ), + "panel-both": lambda d: PanelOLS( + d.y, d[["x"]], entity_effects=True, time_effects=True + ), + "panel-other": lambda d: PanelOLS(d.y, d[["x"]], other_effects=d[["group"]]), + "panel-entity-other": lambda d: PanelOLS( + d.y, d[["x"]], entity_effects=True, other_effects=d[["group"]] + ), +} +FIT_OPTIONS = { + "default": {}, + "robust": {"cov_type": "robust"}, + "kernel": {"cov_type": "kernel"}, + "clustered-entity": {"cov_type": "clustered", "cluster_entity": True}, + "clustered-time": {"cov_type": "clustered", "cluster_time": True}, +} + + +@pytest.mark.parametrize("labels", ["int", "str-date"]) +@pytest.mark.parametrize("level", ["entity", "time"]) +@pytest.mark.parametrize("model", MODELS) +def test_every_estimator_rejects_a_missing_label(model, level, labels): + # LinAlgError (SVD did not converge) was raised by PanelOLS with entity effects + # and a missing entity and by PanelOLS with time effects and a missing time. + # RandomEffects with a missing entity raised "array must not contain infs or + # NaNs" + data = make_panel(level, labels) + with pytest.raises(ValueError, match=MESSAGES[level]) as exc_info: + MODELS[model](data).fit() + assert exc_info.type is ValueError + + +@pytest.mark.parametrize("level", ["entity", "time"]) +@pytest.mark.parametrize("options", FIT_OPTIONS) +@pytest.mark.parametrize("model", ["panel-entity", "panel-both", "random-effects"]) +def test_covariance_estimators(model, options, level): + data = make_panel(level, "int") + with pytest.raises(ValueError, match=MESSAGES[level]): + MODELS[model](data).fit(**FIT_OPTIONS[options]) + + +@pytest.mark.parametrize("level", ["entity", "time"]) +def test_low_memory_demeaning_by_both(level): + data = make_panel(level, "int") + mod = PanelOLS(data.y, data[["x"]], entity_effects=True, time_effects=True) + with pytest.raises(ValueError, match=MESSAGES[level]): + mod.fit(low_memory=True) + + +@pytest.mark.parametrize("level", ["entity", "time"]) +@pytest.mark.parametrize( + ("model", "formula"), + [ + (PanelOLS, "y ~ x + EntityEffects + TimeEffects"), + (PanelOLS, "y ~ x + EntityEffects"), + (RandomEffects, "y ~ 1 + x"), + (PooledOLS, "y ~ 1 + x"), + ], +) +def test_formulas(model, formula, level): + data = make_panel(level, "int") + with pytest.raises(ValueError, match=MESSAGES[level]): + model.from_formula(formula, data).fit() + + +# ---------------------------------------------------------------------------- +# PanelData, whose methods the estimators use +# ---------------------------------------------------------------------------- + + +def weights(panel): + return PanelData(panel.dataframe[["w"]]) + + +ENTITY = ("entity",) +TIME = ("time",) +BOTH = ("entity", "time") +# The levels of the index that each operation groups by, and the operation +OPERATIONS = { + "entity_ids": (ENTITY, lambda p: p.entity_ids), + "time_ids": (TIME, lambda p: p.time_ids), + "demean-entity": (ENTITY, lambda p: p.demean("entity")), + "demean-time": (TIME, lambda p: p.demean("time")), + "demean-both": (BOTH, lambda p: p.demean("both")), + "demean-both-low-memory": (BOTH, lambda p: p.demean("both", low_memory=True)), + "demean-entity-array": (ENTITY, lambda p: p.demean("entity", return_panel=False)), + "demean-entity-weights": (ENTITY, lambda p: p.demean("entity", weights=weights(p))), + "demean-time-weights": (TIME, lambda p: p.demean("time", weights=weights(p))), + "mean-entity": (ENTITY, lambda p: p.mean("entity")), + "mean-time": (TIME, lambda p: p.mean("time")), + "mean-entity-weights": (ENTITY, lambda p: p.mean("entity", weights=weights(p))), + "count-entity": (ENTITY, lambda p: p.count("entity")), + "count-time": (TIME, lambda p: p.count("time")), + "dummies-entity": (ENTITY, lambda p: p.dummies("entity")), + "dummies-time": (TIME, lambda p: p.dummies("time")), + "first_difference": (BOTH, lambda p: p.first_difference()), + "values3d": (BOTH, lambda p: p.values3d), + "panel": (BOTH, lambda p: p.panel), +} +CASES = [ + pytest.param(name, level, id=f"{name}-{level}") + for name, (levels, _) in OPERATIONS.items() + for level in levels +] + + +@pytest.mark.parametrize(("operation", "level"), CASES) +def test_panel_data_operations(operation, level): + # The rows with a missing label are not in an entity or time period, so the + # results have no meaning. Most of these returned data that contain NaN or that + # use the last label for the missing one + data = make_panel(level, "int") + panel = PanelData(data[["y", "x", "w"]]) + with pytest.raises(ValueError, match=MESSAGES[level]) as exc_info: + OPERATIONS[operation][1](panel) + assert exc_info.type is ValueError + + +@pytest.mark.parametrize("operation", OPERATIONS) +def test_panel_data_operations_without_a_missing_label(operation): + panel = PanelData(make_panel(None, "int")[["y", "x", "w"]]) + assert OPERATIONS[operation][1](panel) is not None + + +@pytest.mark.parametrize("model", MODELS) +def test_estimators_without_a_missing_label(model): + res = MODELS[model](make_panel(None, "int")).fit() + assert np.all(np.isfinite(res.params))