diff --git a/linearmodels/panel/data.py b/linearmodels/panel/data.py index f2c5b5ce2a..aecca87421 100644 --- a/linearmodels/panel/data.py +++ b/linearmodels/panel/data.py @@ -158,6 +158,10 @@ class PanelData: Index level 1 is time. The columns are the variables. MultiIndex Series are also accepted and treated as single column MultiIndex DataFrames. + Missing values (NaN) in the entity or time index are not supported. The + methods that group by the index raise a ValueError if the level that they + use has a missing label. + Raises ------ TypeError @@ -270,6 +274,7 @@ def __init__( def panel(self) -> _Panel: """pandas Panel view of data""" if self._panel is None: + self._check_index(0, 1) self._panel = _Panel(self._frame) assert self._panel is not None return self._panel @@ -360,6 +365,30 @@ def entities(self) -> list[linearmodels.typing.Label]: index = self.index return list(index.levels[0][index.codes[0]].unique()) + def _check_index(self, *levels: int) -> None: + """ + Raise a ValueError if a level of the index has missing values (NaN) + + Parameters + ---------- + *levels : int + Levels to check, 0 for the entity and 1 for the time + + Notes + ----- + pandas encodes a missing label as -1 in the codes of the level. The + operations that group by a level have to use this to reject it since + a groupby drops the rows that have a missing label, and the codes + select the last label. + """ + for level in levels: + if np.any(self.index.codes[level] < 0): + name = "entity" if level == 0 else "time" + raise ValueError( + f"The {name} index contains missing values (NaN), which are not " + f"supported and are encoded as -1 in the {name} ids." + ) + @property def entity_ids(self) -> linearmodels.typing.data.AnyIntArray: """ @@ -370,13 +399,8 @@ def entity_ids(self) -> linearmodels.typing.data.AnyIntArray: ndarray 2d array containing entity ids corresponding dataframe view """ - index = self.index - if np.any(index.codes[0] < 0): - raise ValueError( - "The entity index contains missing values (NaN), which are not " - "supported and are encoded as -1 in the entity ids." - ) - return np.asarray(index.codes[0])[:, None] + self._check_index(0) + return np.asarray(self.index.codes[0])[:, None] @property def time_ids(self) -> linearmodels.typing.data.AnyIntArray: @@ -388,13 +412,8 @@ def time_ids(self) -> linearmodels.typing.data.AnyIntArray: ndarray 2d array containing time ids corresponding dataframe view """ - index = self.index - if np.any(index.codes[1] < 0): - raise ValueError( - "The time index contains missing values (NaN), which are not " - "supported and are encoded as -1 in the time ids." - ) - return np.asarray(index.codes[1])[:, None] + self._check_index(1) + return np.asarray(self.index.codes[1])[:, None] def _demean_both_low_mem(self, weights: PanelData | None) -> PanelData: groups = PanelData( @@ -597,6 +616,7 @@ def demean( """ if group not in ("entity", "time", "both"): raise ValueError("Group must be one of 'entity', 'time' or 'both'") + self._check_index(*{"entity": (0,), "time": (1,), "both": (0, 1)}[group]) if group == "both": if not low_memory: return self._demean_both(weights) @@ -654,6 +674,7 @@ def count(self, group: str = "entity") -> DataFrame: Counts according to type. Either (entity by var) or (time by var) """ level = 0 if group == "entity" else 1 + self._check_index(level) reindex = self.entities if group == "entity" else self.time out = self._frame.groupby(level=level).count() @@ -694,6 +715,7 @@ def mean( Data mean according to type. Either (entity by var) or (time by var) """ level = 0 if group == "entity" else 1 + self._check_index(level) if weights is None: mu = self._frame.groupby(level=level).mean() else: @@ -764,6 +786,7 @@ def dummies(self, group: str = "entity", drop_first: bool = False) -> DataFrame: if group not in ("entity", "time"): raise ValueError("Group must be one of 'entity' or 'time'") axis = 0 if group == "entity" else 1 + self._check_index(axis) labels = self.index.codes levels = self.index.levels cat = Categorical(levels[axis][labels[axis]]) diff --git a/linearmodels/tests/panel/test_missing_index.py b/linearmodels/tests/panel/test_missing_index.py new file mode 100644 index 0000000000..11465a513c --- /dev/null +++ b/linearmodels/tests/panel/test_missing_index.py @@ -0,0 +1,228 @@ +""" +Missing values (NaN) in the entity or time index of a panel + +pandas encodes a missing label in a level of an index as -1 in the codes of the +level. Operations that group by the index, such as demeaning by entity, silently +drop the rows with a missing label, and the data that they return contain NaN. An +estimator that uses them fails later with an error from the linear algebra that +does not say what is wrong, e.g., "SVD did not converge". + +Every estimator has to raise the ValueError that says that the entity or the time +index contains missing values. PanelData.entity_ids and PanelData.time_ids do, but +the other operations that use the index did not, and so PanelOLS with entity +effects, PanelOLS with time effects and RandomEffects did not. +""" + +from linearmodels.compat.pandas import ANNUAL_FREQ + +import numpy as np +import pandas as pd +import pytest + +from linearmodels.panel.data import PanelData +from linearmodels.panel.model import ( + BetweenOLS, + FamaMacBeth, + FirstDifferenceOLS, + PanelOLS, + PooledOLS, + RandomEffects, +) + +NENTITY = 20 +NTIME = 5 +MESSAGES = { + "entity": "The entity index contains missing values", + "time": "The time index contains missing values", +} + + +def make_panel(level, labels): + """ + Panel where the label of the entity or the time of one row is missing + + The labels are integers, or strings for the entities and dates for the time. + The panel has no missing labels if level is None. + """ + rng = np.random.default_rng(0) + if labels == "int": + entities = np.repeat(np.arange(NENTITY, dtype=float), NTIME) + times = np.tile(np.arange(NTIME, dtype=float), NENTITY) + missing = np.nan + else: + names = np.array([f"firm{i}" for i in range(NENTITY)], dtype=object) + entities = np.repeat(names, NTIME) + dates = pd.date_range("2001-12-31", periods=NTIME, freq=ANNUAL_FREQ) + times = np.tile(dates.to_numpy(), NENTITY) + missing = None if level == "entity" else np.datetime64("NaT", "ns") + labels_by_level = {"entity": entities, "time": times} + if level is not None: + labels_by_level[level][5] = missing + index = pd.MultiIndex.from_arrays( + [labels_by_level["entity"], labels_by_level["time"]], names=["firm", "year"] + ) + nobs = NENTITY * NTIME + data = pd.DataFrame( + { + "y": rng.standard_normal(nobs), + "x": rng.standard_normal(nobs), + "const": 1.0, + "w": rng.chisquare(5, nobs) + 0.5, + "group": np.arange(nobs) % 4, + }, + index=index, + ) + ncodes = { + name: int(np.sum(data.index.codes[i] == -1)) + for i, name in enumerate(["entity", "time"]) + } + assert ncodes == {"entity": int(level == "entity"), "time": int(level == "time")} + return data + + +MODELS = { + "pooled": lambda d: PooledOLS(d.y, d[["const", "x"]]), + "pooled-weights": lambda d: PooledOLS(d.y, d[["const", "x"]], weights=d.w), + "between": lambda d: BetweenOLS(d.y, d[["const", "x"]]), + "fama-macbeth": lambda d: FamaMacBeth(d.y, d[["const", "x"]]), + "random-effects": lambda d: RandomEffects(d.y, d[["const", "x"]]), + "random-effects-weights": lambda d: RandomEffects( + d.y, d[["const", "x"]], weights=d.w + ), + "first-difference": lambda d: FirstDifferenceOLS(d.y, d[["x"]]), + "panel-no-effects": lambda d: PanelOLS(d.y, d[["const", "x"]]), + "panel-entity": lambda d: PanelOLS(d.y, d[["x"]], entity_effects=True), + "panel-entity-weights": lambda d: PanelOLS( + d.y, d[["x"]], entity_effects=True, weights=d.w + ), + "panel-time": lambda d: PanelOLS(d.y, d[["x"]], time_effects=True), + "panel-time-weights": lambda d: PanelOLS( + d.y, d[["x"]], time_effects=True, weights=d.w + ), + "panel-both": lambda d: PanelOLS( + d.y, d[["x"]], entity_effects=True, time_effects=True + ), + "panel-other": lambda d: PanelOLS(d.y, d[["x"]], other_effects=d[["group"]]), + "panel-entity-other": lambda d: PanelOLS( + d.y, d[["x"]], entity_effects=True, other_effects=d[["group"]] + ), +} +FIT_OPTIONS = { + "default": {}, + "robust": {"cov_type": "robust"}, + "kernel": {"cov_type": "kernel"}, + "clustered-entity": {"cov_type": "clustered", "cluster_entity": True}, + "clustered-time": {"cov_type": "clustered", "cluster_time": True}, +} + + +@pytest.mark.parametrize("labels", ["int", "str-date"]) +@pytest.mark.parametrize("level", ["entity", "time"]) +@pytest.mark.parametrize("model", MODELS) +def test_every_estimator_rejects_a_missing_label(model, level, labels): + # LinAlgError (SVD did not converge) was raised by PanelOLS with entity effects + # and a missing entity and by PanelOLS with time effects and a missing time. + # RandomEffects with a missing entity raised "array must not contain infs or + # NaNs" + data = make_panel(level, labels) + with pytest.raises(ValueError, match=MESSAGES[level]) as exc_info: + MODELS[model](data).fit() + assert exc_info.type is ValueError + + +@pytest.mark.parametrize("level", ["entity", "time"]) +@pytest.mark.parametrize("options", FIT_OPTIONS) +@pytest.mark.parametrize("model", ["panel-entity", "panel-both", "random-effects"]) +def test_covariance_estimators(model, options, level): + data = make_panel(level, "int") + with pytest.raises(ValueError, match=MESSAGES[level]): + MODELS[model](data).fit(**FIT_OPTIONS[options]) + + +@pytest.mark.parametrize("level", ["entity", "time"]) +def test_low_memory_demeaning_by_both(level): + data = make_panel(level, "int") + mod = PanelOLS(data.y, data[["x"]], entity_effects=True, time_effects=True) + with pytest.raises(ValueError, match=MESSAGES[level]): + mod.fit(low_memory=True) + + +@pytest.mark.parametrize("level", ["entity", "time"]) +@pytest.mark.parametrize( + ("model", "formula"), + [ + (PanelOLS, "y ~ x + EntityEffects + TimeEffects"), + (PanelOLS, "y ~ x + EntityEffects"), + (RandomEffects, "y ~ 1 + x"), + (PooledOLS, "y ~ 1 + x"), + ], +) +def test_formulas(model, formula, level): + data = make_panel(level, "int") + with pytest.raises(ValueError, match=MESSAGES[level]): + model.from_formula(formula, data).fit() + + +# ---------------------------------------------------------------------------- +# PanelData, whose methods the estimators use +# ---------------------------------------------------------------------------- + + +def weights(panel): + return PanelData(panel.dataframe[["w"]]) + + +ENTITY = ("entity",) +TIME = ("time",) +BOTH = ("entity", "time") +# The levels of the index that each operation groups by, and the operation +OPERATIONS = { + "entity_ids": (ENTITY, lambda p: p.entity_ids), + "time_ids": (TIME, lambda p: p.time_ids), + "demean-entity": (ENTITY, lambda p: p.demean("entity")), + "demean-time": (TIME, lambda p: p.demean("time")), + "demean-both": (BOTH, lambda p: p.demean("both")), + "demean-both-low-memory": (BOTH, lambda p: p.demean("both", low_memory=True)), + "demean-entity-array": (ENTITY, lambda p: p.demean("entity", return_panel=False)), + "demean-entity-weights": (ENTITY, lambda p: p.demean("entity", weights=weights(p))), + "demean-time-weights": (TIME, lambda p: p.demean("time", weights=weights(p))), + "mean-entity": (ENTITY, lambda p: p.mean("entity")), + "mean-time": (TIME, lambda p: p.mean("time")), + "mean-entity-weights": (ENTITY, lambda p: p.mean("entity", weights=weights(p))), + "count-entity": (ENTITY, lambda p: p.count("entity")), + "count-time": (TIME, lambda p: p.count("time")), + "dummies-entity": (ENTITY, lambda p: p.dummies("entity")), + "dummies-time": (TIME, lambda p: p.dummies("time")), + "first_difference": (BOTH, lambda p: p.first_difference()), + "values3d": (BOTH, lambda p: p.values3d), + "panel": (BOTH, lambda p: p.panel), +} +CASES = [ + pytest.param(name, level, id=f"{name}-{level}") + for name, (levels, _) in OPERATIONS.items() + for level in levels +] + + +@pytest.mark.parametrize(("operation", "level"), CASES) +def test_panel_data_operations(operation, level): + # The rows with a missing label are not in an entity or time period, so the + # results have no meaning. Most of these returned data that contain NaN or that + # use the last label for the missing one + data = make_panel(level, "int") + panel = PanelData(data[["y", "x", "w"]]) + with pytest.raises(ValueError, match=MESSAGES[level]) as exc_info: + OPERATIONS[operation][1](panel) + assert exc_info.type is ValueError + + +@pytest.mark.parametrize("operation", OPERATIONS) +def test_panel_data_operations_without_a_missing_label(operation): + panel = PanelData(make_panel(None, "int")[["y", "x", "w"]]) + assert OPERATIONS[operation][1](panel) is not None + + +@pytest.mark.parametrize("model", MODELS) +def test_estimators_without_a_missing_label(model): + res = MODELS[model](make_panel(None, "int")).fit() + assert np.all(np.isfinite(res.params))