Feature/cd enhancement

.github/workflows/R-CMD-check.yaml

@@ -15,6 +15,10 @@ jobs:
     env:
       R_KEEP_PKG_SOURCE: yes
       R_PROFILE_USER: /dev/null
+      # pak workers read the project profile; CI uses the action's library.
+      RENV_CONFIG_AUTOLOADER_ENABLED: "false"
+      # Allow slower Windows workers up to 60 seconds to start (milliseconds).
+      PKG_SUBPROCESS_TIMEOUT: "60000"
     steps:
       - uses: actions/checkout@v4
       - uses: r-lib/actions/setup-pandoc@v2

NAMESPACE

@@ -2,7 +2,9 @@
 
 S3method(augment,csdm_fit)
 S3method(cd_test,csdm_fit)
+S3method(cd_test,data.frame)
 S3method(cd_test,default)
+S3method(cd_test,pdata.frame)
 S3method(coef,csdm_fit)
 S3method(df.residual,csdm_fit)
 S3method(fitted,csdm_fit)
@@ -13,6 +15,7 @@ S3method(model.matrix,csdm_fit)
 S3method(nobs,csdm_fit)
 S3method(predict,csdm_fit)
 S3method(print,cd_test)
+S3method(print,cd_test_list)
 S3method(print,csdm_fit)
 S3method(print,summary.csdm_fit)
 S3method(residuals,csdm_fit)

NEWS.md

@@ -1,3 +1,24 @@
+# csdm (development version)
+
+- Add `cd_test()` methods for `data.frame` and `pdata.frame` inputs. Select
+  numeric panel variables explicitly using bare names, quoted names, or
+  character vectors. Plain data frames require named `id` and `time` columns;
+  indexed panels use their stored indexes. Return separate named results per
+  variable and print a combined table with variable-specific sample dimensions.
+- Add `reps` to CDw and CDw+, defaulting to 1. Repeated weighted statistics use
+  the sum divided by `sqrt(reps)`, following equation (33) of Juodis and Reese;
+  CDw+ adds its screening term once. Seeded calls continue to preserve RNG state.
+- Record incomplete time periods removed by `drop.incomplete.times` in
+  `excluded_times`, alongside periods with no finite observations.
+- Correct fitted-model CD* input to retain the common-factor component after
+  subtracting the economic and deterministic fitted terms, following
+  Pesaran-Xie's regression input construction. This changes fitted CD* results;
+  CD, CDw, and CDw+ still use full fitted residuals. Preserve and document the
+  standardized-PCA implementation variant for CD*.
+- Clarify the CDw and CDw+ formulas against Juodis-Reese equations (30) and (32),
+  document the complete CD* correction, and correct the Pesaran-Xie and
+  Fan-Liao-Yao references.
+
 # csdm 2.0.0
 
 Version 2.0.0 expands the package's R model interfaces, adds new sample

R/utils_cd_data.R

@@ -0,0 +1,132 @@
+#' @rdname cd_test
+#' @param id,time Column names (strings) for the unit and time indexes of a plain
+#'   data frame. For a \code{pdata.frame}, these are inferred from its stored
+#'   indexes and supplied values are ignored. Time labels need not be numeric.
+#' @export
+#' @method cd_test data.frame
+cd_test.data.frame <- function(object, ..., id = NULL, time = NULL,
+                               type = c("CD", "CDw", "CDw+", "CDstar", "all"),
+                               n_pc = 4L, seed = NULL, min_overlap = 2L,
+                               na.action = c("pairwise", "drop.incomplete.times"),
+                               reps = 1L) {
+  vars <- .cd_select_variables(as.list(substitute(list(...)))[-1L],
+                               names(object), parent.frame())
+  panel <- .cd_panel_input(object, id, time, vars)
+  type <- match.arg(type)
+  na.action <- match.arg(na.action)
+  call <- match.call()
+  results <- lapply(vars, function(variable) {
+    E <- matrix(NA_real_, length(panel$units), length(panel$times),
+                dimnames = list(panel$units, as.character(panel$times)))
+    E[panel$cells] <- as.numeric(object[[variable]])
+    result <- cd_test.default(E, type = type, n_pc = n_pc, seed = seed,
+                              min_overlap = min_overlap, na.action = na.action, reps = reps)
+    result$variable <- variable
+    result$units <- panel$units[!seq_along(panel$units) %in% result$excluded_units]
+    result$excluded_unit_ids <- panel$units[result$excluded_units]
+    result$call <- call
+    result
+  })
+  names(results) <- vars
+  structure(results, class = c("cd_test_list", "list"), call = call,
+            id = panel$id, time = panel$time)
+}
+
+# The data-frame method reads the pdata.frame indexes without requiring plm.
+#' @rdname cd_test
+#' @export
+#' @method cd_test pdata.frame
+cd_test.pdata.frame <- cd_test.data.frame
+
+.cd_select_variables <- function(expressions, columns, envir) {
+  if (!length(expressions)) {
+    stop("Select at least one variable explicitly in '...'.", call. = FALSE)
+  }
+  if (!is.null(names(expressions)) && any(nzchar(names(expressions)))) {
+    stop("Variable selections in '...' must be unnamed; name test controls explicitly.",
+         call. = FALSE)
+  }
+  selections <- lapply(expressions, function(expr) {
+    if (is.symbol(expr) && as.character(expr) %in% columns) {
+      return(as.character(expr))
+    }
+    value <- tryCatch(eval(expr, envir), error = function(e) NULL)
+    if (!is.character(value) || !length(value) || anyNA(value) || any(!nzchar(value))) {
+      stop("Select variables using bare column names, quoted names, or character vectors.",
+           call. = FALSE)
+    }
+    value
+  })
+  vars <- unlist(selections, use.names = FALSE)
+  if (anyDuplicated(vars)) stop("Select each variable only once.", call. = FALSE)
+  unknown <- setdiff(vars, columns)
+  if (length(unknown)) stop("Unknown variable(s): ", paste(unknown, collapse = ", "),
+                            call. = FALSE)
+  vars
+}
+
+.cd_panel_input <- function(object, id, time, vars) {
+  if (!nrow(object) || anyDuplicated(names(object))) {
+    stop("Supply nonempty data with unique column names.", call. = FALSE)
+  }
+  if (inherits(object, "pdata.frame")) {
+    index <- attr(object, "index")
+    if (!is.data.frame(index) || ncol(index) < 2L || nrow(index) != nrow(object)) {
+      stop("The pdata.frame must have valid stored unit and time indexes.", call. = FALSE)
+    }
+    id <- names(index)[1L]
+    time <- names(index)[2L]
+    keys <- index[1:2]
+  } else {
+    if (!is.character(id) || length(id) != 1L || is.na(id) || !nzchar(id) ||
+        !is.character(time) || length(time) != 1L || is.na(time) || !nzchar(time) ||
+        id == time || !all(c(id, time) %in% names(object))) {
+      stop("'id' and 'time' must name distinct columns in the data frame.", call. = FALSE)
+    }
+    keys <- object[c(id, time)]
+  }
+  valid_key <- function(x) {
+    is.atomic(x) && is.null(dim(x)) && !is.complex(x) &&
+      !anyNA(x) && all(nzchar(as.character(x))) &&
+      (!(is.numeric(x) || inherits(x, "Date") || inherits(x, "POSIXt")) ||
+         all(is.finite(x)))
+  }
+  if (!all(vapply(keys, valid_key, logical(1)))) {
+    stop("Panel keys must be nonmissing and finite, with no empty labels.", call. = FALSE)
+  }
+  if (anyDuplicated(keys)) stop("Duplicate id/time cells are not allowed.", call. = FALSE)
+  if (any(vars %in% c(id, time))) {
+    stop("Select test variables other than the unit and time indexes.", call. = FALSE)
+  }
+  numeric_column <- function(x) is.numeric(x) && is.null(dim(x))
+  if (!all(vapply(object[vars], numeric_column, logical(1)))) {
+    stop("Selected test variables must be numeric vectors.", call. = FALSE)
+  }
+  units <- sort(unique(as.character(keys[[1L]])))
+  times <- sort(unique(keys[[2L]]))
+  list(id = id, time = time, units = units, times = times,
+       cells = cbind(match(as.character(keys[[1L]]), units), match(keys[[2L]], times)))
+}
+
+#' @rdname cd_test
+#' @export
+#' @method print cd_test_list
+print.cd_test_list <- function(x, digits = 3, ...) {
+  rows <- lapply(names(x), function(variable) {
+    result <- x[[variable]]
+    selected <- if (identical(result$type, "CDw+")) "CDw_plus" else result$type
+    tests <- result$tests[selected]
+    data.frame(variable = variable, test = gsub("CDw_plus", "CDw+", names(tests), fixed = TRUE),
+               statistic = vapply(tests, function(test) test$statistic, numeric(1)),
+               p.value = vapply(tests, function(test) test$p.value, numeric(1)),
+               N = result$N, T = result$T, row.names = NULL)
+  })
+  cat("Cross-sectional dependence tests by variable\n\n")
+  table <- do.call(rbind, rows)
+  for (column in c("statistic", "p.value")) {
+    table[[column]] <- ifelse(is.na(table[[column]]), "NA",
+                             formatC(table[[column]], digits = digits, format = "f"))
+  }
+  print(table, row.names = FALSE, right = TRUE)
+  invisible(x)
+}

R/utils_residuals.R

@@ -172,8 +172,9 @@ get_residuals <- function(object,
 #' @details
 #' This helper is retained temporarily for compatibility. Row scaling changes
 #' the weighted covariance underlying CDw, while cross-sectional time demeaning
-#' can mechanically induce dependence. Neither transformation is applied by
-#' `cd_test()`.
+#' can mechanically induce dependence. CDw does not standardize unit variances
+#' or demean across units. CD* uses within-unit standardization before PCA,
+#' as described in [cd_test()].
 #'
 #' ## Transformation steps
 #'
@@ -185,9 +186,9 @@ get_residuals <- function(object,
 #'
 #' ## Why this preprocessing matters
 #'
-#' CD-type tests are sensitive to scale heterogeneity and sparse columns in
-#' unbalanced panels. This helper creates a better-conditioned input matrix while
-#' preserving as much usable information as possible.
+#' These transformations change the diagnostic input. Their suitability depends
+#' on the selected test and its assumptions; preprocessing does not establish
+#' the validity of the test's reference distribution.
 #'
 #' @examples
 #' data(PWT_60_07, package = "csdm")

README.md

@@ -11,7 +11,7 @@
 `csdm` estimates heterogeneous panel models when units may share unobserved
 common factors. It provides mean-group (MG), common correlated effects (CCE),
 dynamic CCE (DCCE), and cross-sectionally augmented ARDL (CS-ARDL) estimators,
-along with residual cross-sectional dependence diagnostics.
+along with cross-sectional dependence diagnostics for residuals and panel variables.
 
 The package follows the econometric structure used by Stata's `xtdcce2`, while
 using standard R model methods and explicit specification objects. It does not
@@ -92,13 +92,27 @@ preserves gaps in calendar time.
 
 ## Cross-sectional dependence diagnostics
 
-`cd_test()` accepts a fitted `csdm` model or an `N` by `T` residual matrix:
+`cd_test()` accepts a fitted `csdm` model, an `N` by `T` numeric matrix,
+or explicitly selected variables in a `data.frame` or `pdata.frame`:
 
 ```r
 cd_test(cce, type = "CD")
 cd_test(cce, type = "all", seed = 42)
+
+cd_test(dat, log_rgdpo, "log_hc", id = "id", time = "year")
+
+vars <- c("log_rgdpo", "log_hc")
+cd_test(dat, vars, id = "id", time = "year", type = "CDw",
+        reps = 30, seed = 42)
 ```
 
+Data inputs return a named list of separate results per variable, printed in a
+combined table. Variables must be selected explicitly through `...`, using bare
+names, quoted names, or character vectors. Name `id`, `time`, and all test
+controls; a `pdata.frame` supplies its stored indexes automatically. Each
+variable uses its own available sample. The tests operate on the selected
+observations without fitting a regression.
+
 The available diagnostics are classical CD, randomized CDw, power-enhanced
 CDw+, and bias-corrected CD\*. CD uses pairwise-complete observations by
 default. CDw, CDw+, and CD\* require a balanced residual sample. Periods with no
@@ -111,10 +125,24 @@ cd_test(cce, type = "all", seed = 42,
 ```
 
 Use a fixed `seed` when reporting CDw or CDw+ because their Rademacher weights
-are random. The tests use different corrections and should be interpreted
+are random. `reps = 1` preserves the single-draw behavior. For multiple draws,
+equation (33) of Juodis and Reese combines the CDw statistics as their sum
+divided by `sqrt(reps)`; CDw+ adds its screening term once to that aggregate.
+The authors suggest a modest number of draws, such as 30. Seeded data calls
+apply the seed separately to each variable, so selection order does not affect
+its result. The tests use different corrections and should be interpreted
 against their own assumptions; agreement among p-values is not a substitute
 for checking those assumptions.
 
+CDw follows equation (30) of Juodis and Reese, using weighted covariances and
+the inverse pooled variance of the demeaned input. CDw+ follows their equation
+(32). CD* uses the Pesaran-Xie correction with a standardized-PCA variant:
+the package standardizes units before PCA, whereas the paper uses
+unstandardized observations. For fitted models, CD* removes the economic and
+deterministic fitted component while retaining the common-factor component
+for PCA; the other tests use full fitted residuals. Thus CD* on a fit can differ
+from CD* on `residuals(fit)`. The vignette details these formulas and assumptions.
+
 ## R model interface
 
 Fitted models support the model methods expected by downstream R tools:

inst/REFERENCES.bib

@@ -49,13 +49,14 @@ @article{Ditzen2018
 }
 
 @article{FanLiaoYao2015,
-  author = {Fan, Jianqing and Liao, Yuan and Yao, Jianfeng},
+  author = {Fan, Jianqing and Liao, Yuan and Yao, Jiawei},
   title = {Power Enhancement in High-Dimensional Cross-Section Tests},
   journal = {Econometrica},
   year = {2015},
   volume = {83},
   number = {4},
-  pages = {1497--1541}
+  pages = {1497--1541},
+  doi = {10.3982/ECTA12749}
 }
 
 @article{Pesaran2007,
@@ -89,21 +90,33 @@ @article{Pesaran2021
 }
 
 @article{JuodisReese2021,
-  author = {Juodis, Artūras and Reese, Søren},
-  title = {The incidental parameters problem in testing for remaining cross-sectional correlation},
+  author = {Juodis, Artūras and Reese, Simon},
+  title = {The incidental parameters problem in testing for remaining cross-section correlation},
   journal = {Journal of Business and Economic Statistics},
-  year = {2021},
+  year = {2022},
   volume = {40},
   number = {3},
-  pages = {1191--1203}
+  pages = {1191--1203},
+  doi = {10.1080/07350015.2021.1906687}
 }
 
-@article{PesaranXie2021,
-  author = {Pesaran, M. Hashem and Xie, Yue},
-  title = {A bias-corrected CD test for error cross-sectional dependence in panel models},
-  journal = {Econometric Reviews},
+@techreport{PesaranXie2021,
+  author = {Pesaran, M. Hashem and Xie, Yimeng},
+  title = {A bias-corrected CD test for error cross-sectional dependence in panel data models with latent factors},
+  institution = {CESifo},
+  type = {Working Paper},
+  number = {9234},
   year = {2021},
-  volume = {41},
-  number = {6},
-  pages = {649--677}
+  note = {Revised August 2022},
+  url = {https://www.ifo.de/DocDL/cesifo1_wp9234.pdf}
+}
+
+@article{PesaranXie2026,
+  author = {Pesaran, M. Hashem and Xie, Yimeng},
+  title = {How to detect network dependence in latent factor models? A bias-corrected CD test},
+  journal = {Econometric Theory},
+  year = {2026},
+  pages = {1--41},
+  note = {First View},
+  doi = {10.1017/S0266466625100212}
 }